From 825337b2c08c7c9f132def8699a515598c034962 Mon Sep 17 00:00:00 2001 From: yuyu Date: Mon, 27 Jul 2026 16:33:30 +0800 Subject: [PATCH 01/18] fix(plugin): synchronize Hermes release version metadata --- .../memos-local-plugin-post-merge-dry-run.yml | 36 +++- .../workflows/memos-local-plugin-publish.yml | 117 +++++++++++- .../adapters/hermes/install.hermes.sh | 12 ++ .../hermes/memos_provider/__init__.py | 15 +- .../adapters/hermes/plugin.yaml | 2 +- apps/memos-local-plugin/install.ps1 | 19 +- apps/memos-local-plugin/install.sh | 7 + apps/memos-local-plugin/package.json | 5 +- .../scripts/sync-hermes-version.cjs | 59 ++++++ .../python/test_hermes_provider_pipeline.py | 6 + .../unit/install/hermes-version-sync.test.ts | 168 ++++++++++++++++++ 11 files changed, 433 insertions(+), 13 deletions(-) create mode 100644 apps/memos-local-plugin/scripts/sync-hermes-version.cjs create mode 100644 apps/memos-local-plugin/tests/unit/install/hermes-version-sync.test.ts diff --git a/.github/workflows/memos-local-plugin-post-merge-dry-run.yml b/.github/workflows/memos-local-plugin-post-merge-dry-run.yml index 2be36b13d..b7d015c07 100644 --- a/.github/workflows/memos-local-plugin-post-merge-dry-run.yml +++ b/.github/workflows/memos-local-plugin-post-merge-dry-run.yml @@ -10,17 +10,51 @@ on: - ".github/scripts/draft-local-plugin-release-notes.mjs" - ".github/scripts/draft-local-plugin-release-notes.test.mjs" - ".github/scripts/retry.sh" + - "apps/memos-local-plugin/package.json" + - "apps/memos-local-plugin/package-lock.json" + - "apps/memos-local-plugin/adapters/hermes/plugin.yaml" + - "apps/memos-local-plugin/scripts/sync-hermes-version.cjs" + - "apps/memos-local-plugin/tests/unit/install/hermes-version-sync.test.ts" permissions: contents: write pull-requests: write jobs: + prepare-version: + if: ${{ github.repository == 'MemTensor/MemOS' }} + runs-on: ubuntu-latest + outputs: + version: ${{ steps.version.outputs.version }} + steps: + - uses: actions/checkout@v4 + + - name: Generate dry-run prerelease version + id: version + working-directory: apps/memos-local-plugin + env: + RUN_NUMBER: ${{ github.run_number }} + run: | + version="$( + node -e ' + const current = require("./package.json").version; + const match = current.match(/^(\d+)\.(\d+)\.(\d+)/); + if (!match) { + throw new Error(`Unsupported package version: ${current}`); + } + process.stdout.write( + `${match[1]}.${match[2]}.${Number(match[3]) + 1}-ci.${process.env.RUN_NUMBER}`, + ); + ' + )" + echo "version=${version}" >> "${GITHUB_OUTPUT}" + dry-run: if: ${{ github.repository == 'MemTensor/MemOS' }} + needs: prepare-version uses: ./.github/workflows/memos-local-plugin-publish.yml with: - version: "2.0.10" + version: ${{ needs.prepare-version.outputs.version }} tag: "latest" git_ref: "" release_notes: "" diff --git a/.github/workflows/memos-local-plugin-publish.yml b/.github/workflows/memos-local-plugin-publish.yml index 372e1e240..ecd4322f2 100644 --- a/.github/workflows/memos-local-plugin-publish.yml +++ b/.github/workflows/memos-local-plugin-publish.yml @@ -181,12 +181,67 @@ jobs: exit 1 fi bash ../../.github/scripts/retry.sh --label "bump package version" -- npm version "${RELEASE_VERSION}" --no-git-tag-version --allow-same-version + npm run sync:hermes-version + npm run check:hermes-version - name: Validate and build package + env: + RELEASE_VERSION: ${{ inputs.version }} run: | + set -euo pipefail bash ../../.github/scripts/retry.sh --attempts 2 --label "npm run lint" -- npm run lint bash ../../.github/scripts/retry.sh --attempts 2 --label "npm test" -- npm test - bash ../../.github/scripts/retry.sh --label "npm pack dry-run" -- bash -euo pipefail -c 'npm pack --dry-run --json > "${RUNNER_TEMP}/memos-local-plugin-pack.json"' + release_dir="${RUNNER_TEMP}/memos-local-plugin-release" + pack_json="${RUNNER_TEMP}/memos-local-plugin-pack.json" + mkdir -p "${release_dir}" + bash ../../.github/scripts/retry.sh --label "npm pack" -- \ + bash -euo pipefail -c 'npm pack --json --silent --pack-destination "$1" > "$2"' \ + _ "${release_dir}" "${pack_json}" + + pack_filename="$( + node -e ' + const fs = require("node:fs"); + const raw = fs.readFileSync(process.argv[1], "utf8"); + const jsonStart = raw.match(/^\[/m); + if (!jsonStart || jsonStart.index === undefined) { + throw new Error("npm pack output did not contain a JSON report"); + } + const report = JSON.parse(raw.slice(jsonStart.index)); + if (!Array.isArray(report) || report.length !== 1 || !report[0].filename) { + throw new Error("npm pack did not report exactly one tarball"); + } + fs.writeFileSync(process.argv[1], `${JSON.stringify(report, null, 2)}\n`); + process.stdout.write(report[0].filename); + ' "${pack_json}" + )" + release_tarball="${release_dir}/${pack_filename}" + test -s "${release_tarball}" + + package_version="$( + tar -xOf "${release_tarball}" package/package.json \ + | node -e ' + const fs = require("node:fs"); + process.stdout.write(JSON.parse(fs.readFileSync(0, "utf8")).version); + ' + )" + manifest_version="$( + tar -xOf "${release_tarball}" package/adapters/hermes/plugin.yaml \ + | awk '$1 == "version:" { print $2; exit }' + )" + if [ "${package_version}" != "${RELEASE_VERSION}" ]; then + echo "::error::Tarball package.json version ${package_version} does not match ${RELEASE_VERSION}." + exit 1 + fi + if [ "${manifest_version}" != "${RELEASE_VERSION}" ]; then + echo "::error::Tarball Hermes manifest version ${manifest_version} does not match ${RELEASE_VERSION}." + exit 1 + fi + + sha256sum "${release_tarball}" > "${release_tarball}.sha256" + { + echo "RELEASE_TARBALL=${release_tarball}" + echo "RELEASE_TARBALL_SHA256=${release_tarball}.sha256" + } >> "${GITHUB_ENV}" - name: Draft GitHub Release notes id: release_notes @@ -228,6 +283,8 @@ jobs: cp "${RELEASE_NOTES_FILE}" "${inspection_dir}/release-notes.md" cp "${RUNNER_TEMP}/memos-local-plugin-pack.json" "${inspection_dir}/npm-pack.json" + cp "${RELEASE_TARBALL}" "${inspection_dir}/" + cp "${RELEASE_TARBALL_SHA256}" "${inspection_dir}/" if [ -n "${EVIDENCE_FILE}" ] && [ -s "${EVIDENCE_FILE}" ]; then cp "${EVIDENCE_FILE}" "${inspection_dir}/evidence.json" fi @@ -350,6 +407,54 @@ jobs: sleep "$((attempt * 5))" done } + verify_published_package() { + local verify_dir="${RUNNER_TEMP}/memos-local-plugin-registry-verification" + local verify_json="${verify_dir}/npm-pack.json" + local verify_filename + local verify_tarball + local package_version + local manifest_version + mkdir -p "${verify_dir}" + bash ../../.github/scripts/retry.sh --label "download published npm package" -- \ + bash -euo pipefail -c 'npm pack "$1" --json --silent --pack-destination "$2" > "$3"' \ + _ "${PACKAGE_NAME}@${RELEASE_VERSION}" "${verify_dir}" "${verify_json}" + verify_filename="$( + node -e ' + const fs = require("node:fs"); + const raw = fs.readFileSync(process.argv[1], "utf8"); + const jsonStart = raw.match(/^\[/m); + if (!jsonStart || jsonStart.index === undefined) { + throw new Error("npm pack output did not contain a JSON report"); + } + const report = JSON.parse(raw.slice(jsonStart.index)); + if (!Array.isArray(report) || report.length !== 1 || !report[0].filename) { + throw new Error("npm pack did not report exactly one registry tarball"); + } + fs.writeFileSync(process.argv[1], `${JSON.stringify(report, null, 2)}\n`); + process.stdout.write(report[0].filename); + ' "${verify_json}" + )" + verify_tarball="${verify_dir}/${verify_filename}" + package_version="$( + tar -xOf "${verify_tarball}" package/package.json \ + | node -e ' + const fs = require("node:fs"); + process.stdout.write(JSON.parse(fs.readFileSync(0, "utf8")).version); + ' + )" + manifest_version="$( + tar -xOf "${verify_tarball}" package/adapters/hermes/plugin.yaml \ + | awk '$1 == "version:" { print $2; exit }' + )" + if [ "${package_version}" != "${RELEASE_VERSION}" ]; then + echo "::error::Published package.json version ${package_version} does not match ${RELEASE_VERSION}." + exit 1 + fi + if [ "${manifest_version}" != "${RELEASE_VERSION}" ]; then + echo "::error::Published Hermes manifest version ${manifest_version} does not match ${RELEASE_VERSION}." + exit 1 + fi + } if npm_version_exists; then release_tag="memos-local-plugin-v${RELEASE_VERSION}" @@ -366,7 +471,7 @@ jobs: mkdir -p "${attempt_dir}" for attempt in 1 2 3; do set +e - npm publish --access public --tag "${NPM_DIST_TAG}" >"${attempt_dir}/${attempt}.log" 2>&1 + npm publish "${RELEASE_TARBALL}" --access public --tag "${NPM_DIST_TAG}" >"${attempt_dir}/${attempt}.log" 2>&1 publish_status=$? set -e sed -n '1,160p' "${attempt_dir}/${attempt}.log" @@ -392,6 +497,7 @@ jobs: exit 1 fi fi + verify_published_package - name: Create release tag and PR if: ${{ inputs.dry_run != true }} @@ -501,7 +607,7 @@ jobs: --base "${DEFAULT_BRANCH}" \ --head "${release_branch}" \ --title "${release_title}" \ - --body "Bumps apps/memos-local-plugin/package.json for the published npm release ${RELEASE_VERSION}." + --body "Synchronizes package and Hermes metadata for the published npm release ${RELEASE_VERSION}." status=$? set -e if [ "${status}" = 0 ]; then @@ -519,7 +625,10 @@ jobs: done } - git add apps/memos-local-plugin/package.json apps/memos-local-plugin/package-lock.json + git add \ + apps/memos-local-plugin/package.json \ + apps/memos-local-plugin/package-lock.json \ + apps/memos-local-plugin/adapters/hermes/plugin.yaml created_release_commit=false if ! git diff --staged --quiet; then git commit -m "${release_title}" diff --git a/apps/memos-local-plugin/adapters/hermes/install.hermes.sh b/apps/memos-local-plugin/adapters/hermes/install.hermes.sh index b1c6d8789..a010c68d0 100755 --- a/apps/memos-local-plugin/adapters/hermes/install.hermes.sh +++ b/apps/memos-local-plugin/adapters/hermes/install.hermes.sh @@ -26,6 +26,18 @@ warn() { printf "\033[1;33m[install:hermes]\033[0m %s\n" "$*" >&2; } cd "$PREFIX" +# Keep the Hermes manifest aligned with the npm package before exposing the +# provider directory to the host. Older local source layouts may not contain +# the helper, so retain the packaged manifest as a compatibility fallback. +VERSION_SYNC_SCRIPT="$PREFIX/scripts/sync-hermes-version.cjs" +if command -v node >/dev/null 2>&1 && [[ -f "$VERSION_SYNC_SCRIPT" ]]; then + node "$VERSION_SYNC_SCRIPT" "$PREFIX" >/dev/null +else + warn "Hermes version sync helper unavailable; using packaged plugin.yaml as-is." +fi +cp "$PREFIX/adapters/hermes/plugin.yaml" \ + "$PREFIX/adapters/hermes/memos_provider/plugin.yaml" + # ── 1. node_modules ─────────────────────────────────────────────────────────── if command -v npm >/dev/null 2>&1; then command -v node > .memos-node-bin diff --git a/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py b/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py index 0ff76cf89..c1e56125b 100644 --- a/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py +++ b/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py @@ -88,7 +88,20 @@ class MemoryProvider: # type: ignore[no-redef] logger = logging.getLogger(__name__) PLUGIN_ID = "memos-local-hermes" -PLUGIN_VERSION = "2.0.0-beta.1" + + +def _read_plugin_version() -> str: + """Read the npm package version that owns this Hermes adapter.""" + package_json = _PLUGIN_DIR.parents[2] / "package.json" + try: + payload = json.loads(package_json.read_text(encoding="utf-8")) + except (OSError, ValueError, TypeError): + return "dev" + version = payload.get("version") + return version.strip() if isinstance(version, str) and version.strip() else "dev" + + +PLUGIN_VERSION = _read_plugin_version() _TOOL_FAILURE_REPAIR_HINT = ( "This tool has failed multiple times in a row. You may want to call " "`memos_search` for relevant past experience before deciding what to do next." diff --git a/apps/memos-local-plugin/adapters/hermes/plugin.yaml b/apps/memos-local-plugin/adapters/hermes/plugin.yaml index a5b5607db..3c1171bea 100644 --- a/apps/memos-local-plugin/adapters/hermes/plugin.yaml +++ b/apps/memos-local-plugin/adapters/hermes/plugin.yaml @@ -1,5 +1,5 @@ name: memtensor -version: 2.0.0-beta.1 +version: 2.0.11 description: >- MemOS Local — Reflect2Evolve V7 memory for hermes-agent. Layered L1/L2/L3 traces, reflection-weighted reward backprop, diff --git a/apps/memos-local-plugin/install.ps1 b/apps/memos-local-plugin/install.ps1 index 098b7b4e7..ed8d3a17b 100644 --- a/apps/memos-local-plugin/install.ps1 +++ b/apps/memos-local-plugin/install.ps1 @@ -437,6 +437,20 @@ function Install-Hermes { if (-not $PluginDir -or -not (Test-Path $PluginDir)) { Stop-Die "plugins\memory not found" } + $VersionSyncScript = Join-Path $Prefix "scripts\sync-hermes-version.cjs" + if (Test-Path $VersionSyncScript) { + & node $VersionSyncScript $Prefix + if ($LASTEXITCODE -ne 0) { + Stop-Die "Failed to synchronize Hermes plugin version metadata." + } + } else { + Write-Warn "Hermes version sync helper missing; using packaged plugin.yaml as-is." + } + Copy-Item -Path (Join-Path $AdapterDir "plugin.yaml") -Destination (Join-Path $AdapterDir "memos_provider\plugin.yaml") -ErrorAction SilentlyContinue + if (-not (Test-Path (Join-Path $AdapterDir "memos_provider\plugin.yaml"))) { + Write-Warning "plugin.yaml copy may have failed; verify $AdapterDir\memos_provider\plugin.yaml exists." + } + $UserPluginDir = Join-Path $env:LOCALAPPDATA "hermes\plugins\memory" New-Item -ItemType Directory -Path $UserPluginDir -Force | Out-Null Write-Host "Ensuring user plugin dir: $UserPluginDir" @@ -453,11 +467,6 @@ function Install-Hermes { Write-Warning "Failed to create junction at $Target : $_" } } - Copy-Item -Path (Join-Path $AdapterDir "plugin.yaml") -Destination (Join-Path $AdapterDir "memos_provider\plugin.yaml") -ErrorAction SilentlyContinue - if (-not (Test-Path (Join-Path $AdapterDir "memos_provider\plugin.yaml"))) { - Write-Warning "plugin.yaml copy may have failed; verify $AdapterDir\memos_provider\plugin.yaml exists." - } - if (Test-Path $ConfigFile) { $PyScript = @" import sys, yaml diff --git a/apps/memos-local-plugin/install.sh b/apps/memos-local-plugin/install.sh index 20135e3f7..f74e5e089 100755 --- a/apps/memos-local-plugin/install.sh +++ b/apps/memos-local-plugin/install.sh @@ -734,6 +734,13 @@ except Exception: step "Linking memtensor provider" local user_plugin_dir="${HOME}/.hermes/plugins/memory" mkdir -p "${user_plugin_dir}" + local version_sync="${prefix}/scripts/sync-hermes-version.cjs" + if [[ -f "${version_sync}" ]]; then + node "${version_sync}" "${prefix}" >/dev/null \ + || die "Failed to synchronize Hermes plugin version metadata." + else + warn "Hermes version sync helper missing; using packaged plugin.yaml as-is." + fi # Ensure the provider directory is fully populated before symlinking so # the second symlink (user-level) already points at a complete tree. cp "${adapter_dir}/plugin.yaml" "${adapter_dir}/memos_provider/plugin.yaml" 2>/dev/null || true diff --git a/apps/memos-local-plugin/package.json b/apps/memos-local-plugin/package.json index 49c767f9f..a4272f568 100644 --- a/apps/memos-local-plugin/package.json +++ b/apps/memos-local-plugin/package.json @@ -33,6 +33,7 @@ "templates/README.user.md", "viewer/dist", "scripts/postinstall.cjs", + "scripts/sync-hermes-version.cjs", "install.sh", "install.ps1", "tsconfig.json", @@ -49,7 +50,9 @@ "build": "tsc -p tsconfig.build.json && node scripts/copy-runtime-assets.cjs", "build:viewer": "vite build --config vite.config.ts", "build:package": "npm run build && npm run build:viewer", - "prepack": "npm run build:package", + "prepack": "npm run check:hermes-version && npm run build:package", + "sync:hermes-version": "node scripts/sync-hermes-version.cjs", + "check:hermes-version": "node scripts/sync-hermes-version.cjs --check", "dev": "tsc -p tsconfig.json --watch", "viewer:dev": "vite --config vite.config.ts", "bridge": "tsx bridge.cts", diff --git a/apps/memos-local-plugin/scripts/sync-hermes-version.cjs b/apps/memos-local-plugin/scripts/sync-hermes-version.cjs new file mode 100644 index 000000000..088f5e2b2 --- /dev/null +++ b/apps/memos-local-plugin/scripts/sync-hermes-version.cjs @@ -0,0 +1,59 @@ +#!/usr/bin/env node +"use strict"; + +const fs = require("node:fs"); +const path = require("node:path"); + +const args = process.argv.slice(2); +const checkOnly = args.includes("--check"); +const rootArg = args.find((arg) => arg !== "--check"); +const root = path.resolve(rootArg || path.join(__dirname, "..")); +const packageJsonPath = path.join(root, "package.json"); +const manifestPath = path.join(root, "adapters", "hermes", "plugin.yaml"); + +function fail(message) { + process.stderr.write(`Hermes version sync failed: ${message}\n`); + process.exit(1); +} + +if (!fs.existsSync(packageJsonPath)) { + fail(`package.json not found at ${packageJsonPath}`); +} +if (!fs.existsSync(manifestPath)) { + fail(`Hermes manifest not found at ${manifestPath}`); +} + +let packageJson; +try { + packageJson = JSON.parse(fs.readFileSync(packageJsonPath, "utf8")); +} catch (error) { + fail(`cannot parse ${packageJsonPath}: ${error.message}`); +} + +const version = + typeof packageJson.version === "string" ? packageJson.version.trim() : ""; +if (!version) { + fail(`${packageJsonPath} has no non-empty version`); +} + +const manifest = fs.readFileSync(manifestPath, "utf8"); +const versionLine = /^version:\s*.*$/m; +if (!versionLine.test(manifest)) { + fail(`${manifestPath} has no top-level version field`); +} + +const synchronized = manifest.replace(versionLine, `version: ${version}`); +if (checkOnly) { + if (synchronized !== manifest) { + fail(`manifest version does not match package.json (${version})`); + } + process.stdout.write(`Hermes manifest version is synchronized: ${version}\n`); + process.exit(0); +} + +if (synchronized !== manifest) { + fs.writeFileSync(manifestPath, synchronized, "utf8"); + process.stdout.write(`Synchronized Hermes manifest version to ${version}\n`); +} else { + process.stdout.write(`Hermes manifest version already synchronized: ${version}\n`); +} diff --git a/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py b/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py index 71e3b347b..1deb3e7bf 100644 --- a/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py +++ b/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py @@ -68,6 +68,12 @@ def request(self, method: str, params: dict | None = None, **_kwargs: object) -> class HermesProviderPipelineTests(unittest.TestCase): + def test_plugin_version_matches_package_version(self) -> None: + package_json = _ADAPTER_ROOT.parent.parent / "package.json" + package_version = json.loads(package_json.read_text(encoding="utf-8"))["version"] + + self.assertEqual(memos_provider.PLUGIN_VERSION, package_version) + def test_module_imports_cleanly(self) -> None: """Regression guard for #2096: asserts that ``MemosHttpClient`` is NOT present in ``memos_provider``, since the class was referenced diff --git a/apps/memos-local-plugin/tests/unit/install/hermes-version-sync.test.ts b/apps/memos-local-plugin/tests/unit/install/hermes-version-sync.test.ts new file mode 100644 index 000000000..267c3afe1 --- /dev/null +++ b/apps/memos-local-plugin/tests/unit/install/hermes-version-sync.test.ts @@ -0,0 +1,168 @@ +import { execFileSync, spawnSync } from "node:child_process"; +import { + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; + +const repoRoot = path.resolve(__dirname, "../../.."); +const syncScript = path.join(repoRoot, "scripts", "sync-hermes-version.cjs"); +const temporaryRoots: string[] = []; + +function createFixture(packageVersion: string, manifestVersion: string): string { + const root = mkdtempSync(path.join(tmpdir(), "memos-hermes-version-")); + temporaryRoots.push(root); + mkdirSync(path.join(root, "adapters", "hermes"), { recursive: true }); + writeFileSync( + path.join(root, "package.json"), + JSON.stringify({ name: "@memtensor/memos-local-plugin", version: packageVersion }), + "utf8", + ); + writeFileSync( + path.join(root, "adapters", "hermes", "plugin.yaml"), + `name: memtensor\nversion: ${manifestVersion}\ndescription: test fixture\n`, + "utf8", + ); + return root; +} + +afterEach(() => { + for (const root of temporaryRoots.splice(0)) { + rmSync(root, { recursive: true, force: true }); + } +}); + +describe("Hermes version synchronization", () => { + it("keeps the repository manifest aligned with package.json", () => { + const packageJson = JSON.parse( + readFileSync(path.join(repoRoot, "package.json"), "utf8"), + ) as { version: string }; + const manifest = readFileSync( + path.join(repoRoot, "adapters", "hermes", "plugin.yaml"), + "utf8", + ); + const manifestVersion = manifest.match(/^version:\s*(\S+)\s*$/m)?.[1]; + + expect(manifestVersion).toBe(packageJson.version); + }); + + it("writes package.json version into the Hermes manifest", () => { + const root = createFixture("9.8.7-beta.2", "2.0.0-beta.1"); + + execFileSync(process.execPath, [syncScript, root]); + + const manifest = readFileSync( + path.join(root, "adapters", "hermes", "plugin.yaml"), + "utf8", + ); + expect(manifest).toContain("version: 9.8.7-beta.2"); + expect(manifest).not.toContain("version: 2.0.0-beta.1"); + }); + + it("fails check mode for stale metadata and passes after synchronization", () => { + const root = createFixture("3.4.5", "3.4.4"); + + const staleCheck = spawnSync(process.execPath, [syncScript, "--check", root]); + expect(staleCheck.status).not.toBe(0); + + execFileSync(process.execPath, [syncScript, root]); + + const synchronizedCheck = spawnSync(process.execPath, [ + syncScript, + "--check", + root, + ]); + expect(synchronizedCheck.status).toBe(0); + }); + + it("packages the synchronization helper and keeps prepack read-only", () => { + const packageJson = JSON.parse( + readFileSync(path.join(repoRoot, "package.json"), "utf8"), + ) as { files: string[]; scripts: Record }; + + expect(packageJson.files).toContain("scripts/sync-hermes-version.cjs"); + expect(packageJson.scripts["sync:hermes-version"]).toBe( + "node scripts/sync-hermes-version.cjs", + ); + expect(packageJson.scripts.prepack).toBe( + "npm run check:hermes-version && npm run build:package", + ); + }); + + it("synchronizes before tests and publishes the validated tarball", () => { + const workflow = readFileSync( + path.resolve(repoRoot, "../../.github/workflows/memos-local-plugin-publish.yml"), + "utf8", + ); + + const bumpPosition = workflow.indexOf('npm version "${RELEASE_VERSION}"'); + const syncPosition = workflow.indexOf("npm run sync:hermes-version"); + const checkPosition = workflow.indexOf("npm run check:hermes-version"); + const testPosition = workflow.indexOf('"npm test"'); + + expect(bumpPosition).toBeGreaterThan(-1); + expect(syncPosition).toBeGreaterThan(bumpPosition); + expect(checkPosition).toBeGreaterThan(syncPosition); + expect(testPosition).toBeGreaterThan(checkPosition); + expect(workflow).toContain("npm pack --json --silent --pack-destination"); + expect(workflow).toContain("raw.match(/^\\[/m)"); + expect(workflow).toContain( + "fs.writeFileSync(process.argv[1], `${JSON.stringify(report, null, 2)}\\n`)", + ); + expect(workflow).not.toContain("npm pack --dry-run"); + expect(workflow).toContain('npm publish "${RELEASE_TARBALL}"'); + expect(workflow).toContain( + "tar -xOf \"${release_tarball}\" package/adapters/hermes/plugin.yaml", + ); + expect(workflow).toContain("verify_published_package"); + expect(workflow).toMatch( + /git add \\\n\s+apps\/memos-local-plugin\/package\.json \\\n\s+apps\/memos-local-plugin\/package-lock\.json \\\n\s+apps\/memos-local-plugin\/adapters\/hermes\/plugin\.yaml/, + ); + }); + + it("uses a generated version for the post-merge release dry run", () => { + const workflow = readFileSync( + path.resolve( + repoRoot, + "../../.github/workflows/memos-local-plugin-post-merge-dry-run.yml", + ), + "utf8", + ); + + expect(workflow).not.toContain('version: "2.0.10"'); + expect(workflow).toContain("needs.prepare-version.outputs.version"); + expect(workflow).toContain( + "apps/memos-local-plugin/scripts/sync-hermes-version.cjs", + ); + }); + + it("runs synchronization in every Hermes installer", () => { + const unixInstaller = readFileSync(path.join(repoRoot, "install.sh"), "utf8"); + const adapterInstaller = readFileSync( + path.join(repoRoot, "adapters", "hermes", "install.hermes.sh"), + "utf8", + ); + const windowsInstaller = readFileSync( + path.join(repoRoot, "install.ps1"), + "utf8", + ); + + expect(unixInstaller).toContain( + 'version_sync="${prefix}/scripts/sync-hermes-version.cjs"', + ); + expect(unixInstaller).toContain('node "${version_sync}" "${prefix}"'); + expect(adapterInstaller).toContain( + 'VERSION_SYNC_SCRIPT="$PREFIX/scripts/sync-hermes-version.cjs"', + ); + expect(adapterInstaller).toContain('node "$VERSION_SYNC_SCRIPT" "$PREFIX"'); + expect(windowsInstaller).toContain( + '$VersionSyncScript = Join-Path $Prefix "scripts\\sync-hermes-version.cjs"', + ); + expect(windowsInstaller).toContain("& node $VersionSyncScript $Prefix"); + }); +}); From 7ff7c9099aa4e02c41857b86c50bf76f0b30efbd Mon Sep 17 00:00:00 2001 From: geyunhang Date: Mon, 27 Jul 2026 19:48:57 +0800 Subject: [PATCH 02/18] fix(plugin): keep Hermes compression turn read-only --- .../hermes/memos_provider/__init__.py | 80 +++++++++++++------ apps/memos-local-plugin/agent-contract/dto.ts | 6 ++ apps/memos-local-plugin/bridge/methods.ts | 6 ++ .../core/pipeline/orchestrator.ts | 57 +++++++++++++ .../python/test_hermes_provider_pipeline.py | 47 ++++++++++- .../tests/unit/bridge/methods.test.ts | 11 +++ .../tests/unit/pipeline/orchestrator.test.ts | 56 +++++++++++++ 7 files changed, 235 insertions(+), 28 deletions(-) diff --git a/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py b/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py index c1e56125b..d0c9bdb36 100644 --- a/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py +++ b/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py @@ -65,8 +65,8 @@ if str(_PLUGIN_DIR) not in sys.path: sys.path.insert(0, str(_PLUGIN_DIR)) -from bridge_client import BridgeError, MemosBridgeClient # noqa: E402 -from daemon_manager import ( # noqa: E402 +from bridge_client import BridgeError, MemosBridgeClient +from daemon_manager import ( ensure_bridge_running, ensure_viewer_daemon, kill_zombie_bridges, @@ -107,6 +107,7 @@ def _read_plugin_version() -> str: "`memos_search` for relevant past experience before deciding what to do next." ) _TOOL_FAILURE_HINT_THRESHOLD = 3 +_COMPRESSION_CONTEXT_MAX_CHARS = 6_000 def _long_rpc_timeout_default() -> float: @@ -305,6 +306,12 @@ def __init__(self) -> None: self._prefetch_lock = threading.Lock() self._prefetch_result: str = "" self._prefetch_thread: threading.Thread | None = None + # Exact memory context injected by the most recent real prefetch. + # Compression hooks only read this cache; they must never call the + # lifecycle-mutating `turn.start` RPC. + self._state_lock = threading.Lock() + self._last_injected_context: str = "" + self._active_turn_key: str = "" # Tool calls accumulated via the Hermes `post_tool_call` plugin # hook — flushed alongside user/assistant text in `sync_turn`. self._tool_calls: list[dict[str, Any]] = [] @@ -379,6 +386,9 @@ def initialize(self, session_id: str, **kwargs: Any) -> None: # type: ignore[ov self._bridge = None self._session_id = session_id or self._session_id + with self._state_lock: + self._last_injected_context = "" + self._active_turn_key = "" self._hermes_home = str(kwargs.get("hermes_home") or "") self._platform = str(kwargs.get("platform") or "cli") self._agent_identity = str(kwargs.get("agent_identity") or "hermes") @@ -940,6 +950,13 @@ def on_turn_start(self, turn_number: int, message: str, **_kwargs: Any) -> None: self._turn_number = int(turn_number or 0) self._skip_current_turn = _is_hermes_internal_review_prompt(message) self._last_user_text = "" if self._skip_current_turn else (message or "").strip() + with self._state_lock: + self._last_injected_context = "" + self._active_turn_key = ( + f"{self._session_id}:{self._turn_number}" + if self._session_id and self._turn_number > 0 + else "" + ) # Reset per-turn buffers so reasoning / tool calls captured here # belong only to this turn. self._turn_thinking = "" @@ -971,7 +988,11 @@ def prefetch(self, query: str, *, session_id: str = "") -> str: # type: ignore[ if suppress_injection: # Do not let remembered "do it directly" skills override an # explicit user request to dispatch work to a subagent. + with self._state_lock: + self._last_injected_context = "" return "" + with self._state_lock: + self._last_injected_context = context[:_COMPRESSION_CONTEXT_MAX_CHARS] return context except Exception as err: logger.debug("MemOS: prefetch failed — %s", err) @@ -1217,19 +1238,19 @@ def _json_or_raw(value: Any) -> Any: return value def on_pre_compress(self, messages: list[dict[str, Any]]) -> str: # type: ignore[override] - """Extract a compression-time memory summary. + """Return the memory context injected by the current/previous turn. - Hermes calls this right before discarding old messages; we - surface a tight summary of the relevant retrieval packet so - the compressor can preserve it alongside its own summary. + This hook must stay read-only. Calling ``turn.start`` here used to + finalize the active lightweight episode and create a phantom episode + on every compression pass. Reusing the exact context previously + injected by ``prefetch`` also keeps the summary faithful to what the + model actually saw. """ - if not self._bridge or not self._last_user_text: + with self._state_lock: + context = self._last_injected_context.strip() + if not context: return "" - with contextlib.suppress(Exception): - packet = self._turn_start(self._last_user_text, session_id=self._session_id) - if packet: - return f"MemOS memory snapshot (preserved across compression):\n{packet}" - return "" + return f"MemOS memory snapshot (preserved across compression):\n{context}" # ─── Tool surface ───────────────────────────────────────────────────── @@ -1604,6 +1625,9 @@ def save_config(self, values: dict[str, Any], hermes_home: str) -> None: # type # ─── Session-end ────────────────────────────────────────────────────── def on_session_end(self, messages: list[dict[str, Any]]) -> None: # type: ignore[override] + with self._state_lock: + self._last_injected_context = "" + self._active_turn_key = "" if not self._bridge: return # `sync_turn` already flushed completed turn data synchronously. @@ -1640,6 +1664,9 @@ def __del__(self) -> None: self.shutdown() def shutdown(self) -> None: # type: ignore[override] + with self._state_lock: + self._last_injected_context = "" + self._active_turn_key = "" self._bridge_keepalive_stop.set() if self._bridge_keepalive_thread and self._bridge_keepalive_thread.is_alive(): self._bridge_keepalive_thread.join( @@ -2032,20 +2059,25 @@ def _run() -> None: def _turn_start(self, query: str, *, session_id: str = "") -> str: assert self._bridge is not None host_runtime = self._host_runtime_context() - resp = self._bridge.request( - "turn.start", - { - "agent": "hermes", + with self._state_lock: + turn_key = self._active_turn_key + payload: dict[str, Any] = { + "agent": "hermes", + "namespace": self._runtime_namespace(), + "sessionId": session_id or self._session_id, + "userText": query, + "contextHints": { + "agentIdentity": self._agent_identity, "namespace": self._runtime_namespace(), - "sessionId": session_id or self._session_id, - "userText": query, - "contextHints": { - "agentIdentity": self._agent_identity, - "namespace": self._runtime_namespace(), - **host_runtime, - }, - "ts": int(time.time() * 1000), + **host_runtime, }, + "ts": int(time.time() * 1000), + } + if turn_key: + payload["turnKey"] = turn_key + resp = self._bridge.request( + "turn.start", + payload, timeout=_LONG_RPC_TIMEOUT, ) # Stash the real episode id the pipeline auto-created (V7 diff --git a/apps/memos-local-plugin/agent-contract/dto.ts b/apps/memos-local-plugin/agent-contract/dto.ts index e9fe8db90..bdb9375ab 100644 --- a/apps/memos-local-plugin/agent-contract/dto.ts +++ b/apps/memos-local-plugin/agent-contract/dto.ts @@ -89,6 +89,12 @@ export interface TurnInputDTO { agent: AgentKind; sessionId: SessionId; namespace?: RuntimeNamespace; + /** + * Optional host-stable idempotency key for this logical turn. + * Retries with the same sessionId + turnKey must reuse the original + * episode instead of opening another one. + */ + turnKey?: string; /** Optional pre-existing episodeId (for continued tasks). */ episodeId?: EpisodeId; /** Free-form text the user said this turn. */ diff --git a/apps/memos-local-plugin/bridge/methods.ts b/apps/memos-local-plugin/bridge/methods.ts index 1f3583746..2bc1084a6 100644 --- a/apps/memos-local-plugin/bridge/methods.ts +++ b/apps/memos-local-plugin/bridge/methods.ts @@ -383,6 +383,12 @@ function validateTurnInput(p: Record): void { requireKey(p, "sessionId", "string", "turn.start"); requireKey(p, "userText", "string", "turn.start"); requireKey(p, "ts", "number", "turn.start"); + if (p.turnKey !== undefined && typeof p.turnKey !== "string") { + throw new MemosError( + "invalid_argument", + "turn.start: optional 'turnKey' must be a string", + ); + } } function validateTurnResult(p: Record): void { diff --git a/apps/memos-local-plugin/core/pipeline/orchestrator.ts b/apps/memos-local-plugin/core/pipeline/orchestrator.ts index 62e44e017..ad6fc3c44 100644 --- a/apps/memos-local-plugin/core/pipeline/orchestrator.ts +++ b/apps/memos-local-plugin/core/pipeline/orchestrator.ts @@ -77,6 +77,7 @@ import type { } from "../retrieval/types.js"; import type { CoreEvent, CoreEventType } from "../../agent-contract/events.js"; import type { LogRecord } from "../../agent-contract/log-record.js"; +import { ERROR_CODES, MemosError } from "../../agent-contract/errors.js"; import { memoryBuffer } from "../logger/index.js"; import { onBroadcastLog } from "../logger/transports/sse-broadcast.js"; import { createEmbeddingRetryWorker, systemErrorEvent } from "../embedding/index.js"; @@ -300,6 +301,19 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { // when the adapter doesn't pass its own. const lastUserTextBySession = new Map(); + // Keep one idempotent turn-start promise per session. Storing the promise, + // rather than only the resolved packet, also collapses concurrent retries + // that arrive while the first retrieval is still running. A new turnKey + // replaces the prior entry, so this stays bounded to O(active sessions). + const turnStartBySession = new Map< + SessionId, + { + turnKey: string; + userText: string; + packet: Promise; + } + >(); + // When a session is closed (e.g. adapter fires `session_end`), purge // every orchestrator-local map entry for that session. Without this, // `openEpisodeIfNeeded` would still see the stale `lastEpisodeBySession` @@ -312,6 +326,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { openEpisodeBySession.delete(sid); lastEpisodeBySession.delete(sid); lastUserTextBySession.delete(sid); + turnStartBySession.delete(sid); log.debug("session.maps_cleared", { sessionId: sid, reason: evt.reason }); }); @@ -1117,6 +1132,48 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { // ─── Turn lifecycle ───────────────────────────────────────────────────── async function onTurnStart(input: TurnInputDTO): Promise { + const turnKey = input.turnKey?.trim(); + if (!turnKey) { + return onTurnStartOnce(input); + } + + const existing = turnStartBySession.get(input.sessionId); + if (existing?.turnKey === turnKey) { + if (existing.userText !== input.userText) { + throw new MemosError( + ERROR_CODES.CONFLICT, + "turn.start: turnKey was reused with different user text", + { + sessionId: input.sessionId, + turnKey, + }, + ); + } + log.debug("turn.start.idempotent_reuse", { + sessionId: input.sessionId, + turnKey, + }); + return existing.packet; + } + + const packet = onTurnStartOnce(input); + turnStartBySession.set(input.sessionId, { + turnKey, + userText: input.userText, + packet, + }); + try { + return await packet; + } catch (err) { + const current = turnStartBySession.get(input.sessionId); + if (current?.packet === packet) { + turnStartBySession.delete(input.sessionId); + } + throw err; + } + } + + async function onTurnStartOnce(input: TurnInputDTO): Promise { const t0 = now(); const initialSessionId = await ensureSession( input.agent, diff --git a/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py b/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py index 1deb3e7bf..887ccafbe 100644 --- a/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py +++ b/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py @@ -22,7 +22,7 @@ if str(_p) not in sys.path: sys.path.insert(0, str(_p)) -import memos_provider # noqa: E402 +import memos_provider class FakeBridge: @@ -385,7 +385,7 @@ def test_internal_hermes_review_prompt_is_not_persisted_as_user_turn(self) -> No self.assertFalse(any(method == "turn.start" for method, _ in bridge.calls)) self.assertFalse(any(method == "turn.end" for method, _ in bridge.calls)) - def test_on_pre_compress_reuses_last_user_text_for_snapshot(self) -> None: + def test_on_pre_compress_reuses_cached_prefetch_without_starting_turn(self) -> None: bridge = FakeBridge() with ( patch("memos_provider.ensure_bridge_running", return_value=True), @@ -395,13 +395,52 @@ def test_on_pre_compress_reuses_last_user_text_for_snapshot(self) -> None: provider = memos_provider.MemTensorProvider() provider.initialize("compress-session") provider.on_turn_start(2, "compress HERMES_MEMOS_E2E_0428 context") + provider.prefetch("compress HERMES_MEMOS_E2E_0428 context") + episode_id = provider._episode_id + turn_starts_before = sum(method == "turn.start" for method, _ in bridge.calls) snapshot = provider.on_pre_compress([{"role": "user", "content": "x"}]) + repeated = provider.on_pre_compress([{"role": "user", "content": "x"}]) self.assertIn("MemOS memory snapshot", snapshot) self.assertIn("remembered HERMES_MEMOS_E2E_0428", snapshot) - self.assertEqual(bridge.calls[-1][0], "turn.start") - self.assertIn("HERMES_MEMOS_E2E_0428", bridge.calls[-1][1]["userText"]) + self.assertEqual(repeated, snapshot) + self.assertEqual(provider._episode_id, episode_id) + self.assertEqual( + sum(method == "turn.start" for method, _ in bridge.calls), + turn_starts_before, + ) + + def test_on_pre_compress_without_cached_prefetch_is_read_only(self) -> None: + bridge = FakeBridge() + with ( + patch("memos_provider.ensure_bridge_running", return_value=True), + patch("memos_provider.ensure_viewer_daemon", return_value=True), + patch("memos_provider.MemosBridgeClient", return_value=bridge), + ): + provider = memos_provider.MemTensorProvider() + provider.initialize("compress-session") + provider.on_turn_start(1, "first turn") + + snapshot = provider.on_pre_compress([{"role": "user", "content": "first turn"}]) + + self.assertEqual(snapshot, "") + self.assertFalse(any(method == "turn.start" for method, _ in bridge.calls)) + + def test_prefetch_passes_stable_turn_key_to_bridge(self) -> None: + bridge = FakeBridge() + with ( + patch("memos_provider.ensure_bridge_running", return_value=True), + patch("memos_provider.ensure_viewer_daemon", return_value=True), + patch("memos_provider.MemosBridgeClient", return_value=bridge), + ): + provider = memos_provider.MemTensorProvider() + provider.initialize("turn-key-session") + provider.on_turn_start(7, "continue the task") + provider.prefetch("continue the task") + + turn_start = next(params for method, params in bridge.calls if method == "turn.start") + self.assertEqual(turn_start["turnKey"], "turn-key-session:7") def test_prefetch_suppresses_memory_injection_for_explicit_delegation(self) -> None: bridge = FakeBridge() diff --git a/apps/memos-local-plugin/tests/unit/bridge/methods.test.ts b/apps/memos-local-plugin/tests/unit/bridge/methods.test.ts index 2274a704a..8ee5fa921 100644 --- a/apps/memos-local-plugin/tests/unit/bridge/methods.test.ts +++ b/apps/memos-local-plugin/tests/unit/bridge/methods.test.ts @@ -333,6 +333,17 @@ describe("makeDispatcher", () => { ).rejects.toSatisfy( (err) => err instanceof MemosError && err.code === "invalid_argument", ); + await expect( + dispatch("turn.start", { + agent: "openclaw", + sessionId: "s-1", + turnKey: 42, + userText: "hi", + ts: 123, + }), + ).rejects.toSatisfy( + (err) => err instanceof MemosError && err.code === "invalid_argument", + ); }); it("feedback.submit forwards the DTO shape intact", async () => { diff --git a/apps/memos-local-plugin/tests/unit/pipeline/orchestrator.test.ts b/apps/memos-local-plugin/tests/unit/pipeline/orchestrator.test.ts index 2e69522d8..a70fdcf6b 100644 --- a/apps/memos-local-plugin/tests/unit/pipeline/orchestrator.test.ts +++ b/apps/memos-local-plugin/tests/unit/pipeline/orchestrator.test.ts @@ -232,6 +232,62 @@ describe("pipeline/orchestrator", () => { expect(stats?.scenarioId).toBe("CHITCHAT"); }); + it("makes repeated turn.start calls idempotent by turnKey in lightweight mode", async () => { + pipeline = createPipeline({ + ...buildDeps(dbHandle!), + config: configWithLightweightMemory(true), + }); + const input: TurnInputDTO = { + agent: "hermes", + sessionId: "s-idempotent", + turnKey: "s-idempotent:4", + userText: "continue the same real task", + ts: 1_700_000_000_000, + }; + + const [first, concurrent] = await Promise.all([ + pipeline.onTurnStart(input), + pipeline.onTurnStart({ + ...input, + ts: input.ts + 1, + }), + ]); + const repeated = await pipeline.onTurnStart({ + ...input, + ts: input.ts + 2, + }); + + expect(concurrent.episodeId).toBe(first.episodeId); + expect(repeated.episodeId).toBe(first.episodeId); + expect(dbHandle!.repos.episodes.list({ sessionId: input.sessionId })).toHaveLength(1); + expect(pipeline.sessionManager.getEpisode(first.episodeId)?.turns).toHaveLength(1); + }); + + it("rejects a reused turnKey carrying different user text", async () => { + pipeline = createPipeline({ + ...buildDeps(dbHandle!), + config: configWithLightweightMemory(true), + }); + const input: TurnInputDTO = { + agent: "hermes", + sessionId: "s-turn-key-conflict", + turnKey: "s-turn-key-conflict:2", + userText: "first task text", + ts: 1_700_000_000_000, + }; + + await pipeline.onTurnStart(input); + + await expect( + pipeline.onTurnStart({ + ...input, + userText: "different task text", + ts: input.ts + 1, + }), + ).rejects.toMatchObject({ code: "conflict" }); + expect(dbHandle!.repos.episodes.list({ sessionId: input.sessionId })).toHaveLength(1); + }); + it("records tool success + failure through the feedback subscriber", async () => { pipeline = createPipeline(buildDeps(dbHandle!)); await pipeline.onTurnStart({ From f59f78af4379affac1207b9130377128829632c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=81=E5=9C=A8=E5=90=B5=E7=9D=80=E5=90=83=E7=B3=96?= Date: Tue, 28 Jul 2026 16:22:10 +0800 Subject: [PATCH 03/18] fix(plugin): reload Hermes config on restart --- .../memos-local-plugin/server/routes/admin.ts | 46 ++++++++------ .../tests/unit/server/admin.test.ts | 62 +++++++++++++++++++ .../tests/unit/viewer/restart.test.ts | 53 ++++++++++++++++ .../viewer/src/stores/restart.ts | 26 ++++---- 4 files changed, 157 insertions(+), 30 deletions(-) create mode 100644 apps/memos-local-plugin/tests/unit/server/admin.test.ts create mode 100644 apps/memos-local-plugin/tests/unit/viewer/restart.test.ts diff --git a/apps/memos-local-plugin/server/routes/admin.ts b/apps/memos-local-plugin/server/routes/admin.ts index a8daad553..29d38bcde 100644 --- a/apps/memos-local-plugin/server/routes/admin.ts +++ b/apps/memos-local-plugin/server/routes/admin.ts @@ -12,9 +12,8 @@ * Agent-aware restart. For OpenClaw the plugin lives inside the * gateway process, which is managed by macOS launchd — calling * `process.exit(0)` causes launchd to respawn it automatically. - * For Hermes the HTTP viewer is the long-lived smoke/daemon bridge. - * Do NOT restart this process; terminate the active `hermes chat` - * process so the user can relaunch it and reconnect to this viewer. + * For Hermes, terminate the active `hermes chat`, spawn a replacement + * viewer daemon, then exit so both processes reload the saved config. */ import { spawn } from "node:child_process"; import type { ServerDeps, ServerOptions } from "../types.js"; @@ -46,20 +45,7 @@ export function registerAdminRoutes(routes: Routes, deps: ServerDeps, options: S } if (agent !== "openclaw") { // Hermes: spawn replacement daemon after clearing data - const nodePath = await import("node:path"); - const { fileURLToPath } = await import("node:url"); - const { spawn } = await import("node:child_process"); - const thisFile = fileURLToPath(import.meta.url); - const pluginRoot = nodePath.resolve(nodePath.dirname(thisFile), "../.."); - const tsxBin = nodePath.join(pluginRoot, "node_modules/.bin/tsx"); - const bridgeScript = nodePath.join(pluginRoot, "bridge.cts"); - const cmd = `sleep 3 && "${process.execPath}" "${tsxBin}" "${bridgeScript}" --agent=${agent} --daemon`; - const child = spawn("bash", ["-c", cmd], { - detached: true, - stdio: "ignore", - cwd: pluginRoot, - }); - child.unref(); + await spawnReplacementDaemon(agent); } setTimeout(() => process.exit(0), 200); return { ok: true, restarting: true, killedHermes }; @@ -74,13 +60,37 @@ export function registerAdminRoutes(routes: Routes, deps: ServerDeps, options: S if (agent === "hermes") { const killed = await terminateHermesChat(); - return { ok: true, restarting: false, killed }; + await spawnReplacementDaemon(agent); + setTimeout(() => process.exit(0), 200); + return { ok: true, restarting: true, killed }; } return { ok: false, error: `restart unsupported for agent: ${agent}` }; }); } +async function spawnReplacementDaemon(agent: string): Promise { + const fs = await import("node:fs"); + const nodePath = await import("node:path"); + const { fileURLToPath } = await import("node:url"); + const thisFile = fileURLToPath(import.meta.url); + let pluginRoot = nodePath.resolve(nodePath.dirname(thisFile), "../.."); + // Source: /server/routes/admin.ts. Built package: + // /dist/server/routes/admin.js. + if (!fs.existsSync(nodePath.join(pluginRoot, "package.json"))) { + pluginRoot = nodePath.resolve(pluginRoot, ".."); + } + const tsxBin = nodePath.join(pluginRoot, "node_modules/.bin/tsx"); + const bridgeScript = nodePath.join(pluginRoot, "bridge.cts"); + const cmd = `sleep 3 && "${process.execPath}" "${tsxBin}" "${bridgeScript}" --agent=${agent} --daemon`; + const child = spawn("bash", ["-c", cmd], { + detached: true, + stdio: "ignore", + cwd: pluginRoot, + }); + child.unref(); +} + async function terminateHermesChat(): Promise { // Match the Hermes CLI wrapper used by install.sh without touching // `bridge.cts --daemon`, which owns the Memory Viewer port. diff --git a/apps/memos-local-plugin/tests/unit/server/admin.test.ts b/apps/memos-local-plugin/tests/unit/server/admin.test.ts new file mode 100644 index 000000000..6eb61c543 --- /dev/null +++ b/apps/memos-local-plugin/tests/unit/server/admin.test.ts @@ -0,0 +1,62 @@ +import { EventEmitter } from "node:events"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import type { MemoryCore } from "../../../agent-contract/memory-core.js"; +import { registerAdminRoutes } from "../../../server/routes/admin.js"; +import { Routes } from "../../../server/routes/registry.js"; + +const { spawnMock } = vi.hoisted(() => ({ + spawnMock: vi.fn(), +})); + +vi.mock("node:child_process", () => ({ + spawn: spawnMock, +})); + +describe("admin lifecycle routes", () => { + beforeEach(() => { + vi.useFakeTimers(); + spawnMock.mockReset(); + spawnMock.mockImplementation((command: string) => { + const child = new EventEmitter() as EventEmitter & { unref: ReturnType }; + child.unref = vi.fn(); + if (command === "pkill") { + queueMicrotask(() => child.emit("exit", 1)); + } + return child; + }); + }); + + afterEach(() => { + vi.clearAllTimers(); + vi.useRealTimers(); + }); + + it("replaces the Hermes viewer daemon so saved model config takes effect", async () => { + const routes = new Routes(); + registerAdminRoutes( + routes, + { core: {} as MemoryCore }, + { agent: "hermes" }, + ); + + const restart = routes.getExact("POST /api/v1/admin/restart"); + expect(restart).toBeDefined(); + + const result = await restart!({} as never); + + expect(result).toMatchObject({ ok: true, restarting: true }); + expect(spawnMock).toHaveBeenCalledWith( + "bash", + [ + "-c", + expect.stringContaining("--agent=hermes --daemon"), + ], + expect.objectContaining({ + detached: true, + stdio: "ignore", + }), + ); + }); +}); diff --git a/apps/memos-local-plugin/tests/unit/viewer/restart.test.ts b/apps/memos-local-plugin/tests/unit/viewer/restart.test.ts new file mode 100644 index 000000000..88e8e426e --- /dev/null +++ b/apps/memos-local-plugin/tests/unit/viewer/restart.test.ts @@ -0,0 +1,53 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const fakeWindow = { + location: { + pathname: "/", + href: "", + }, +}; + +(globalThis as any).window = fakeWindow; +(globalThis as any).localStorage = { + getItem() { return null; }, +}; + +import { health } from "../../../viewer/src/stores/health"; +import { triggerRestart } from "../../../viewer/src/stores/restart"; + +describe("viewer restart flow", () => { + const originalFetch = globalThis.fetch; + + beforeEach(() => { + vi.useFakeTimers(); + fakeWindow.location.href = ""; + health.value = { ok: true, agent: "hermes" }; + }); + + afterEach(() => { + vi.clearAllTimers(); + vi.useRealTimers(); + globalThis.fetch = originalFetch; + }); + + it("waits for the replacement Hermes daemon before reloading", async () => { + let healthChecks = 0; + globalThis.fetch = vi.fn(async (_url: string | URL | Request, init?: RequestInit) => { + if (init?.method === "POST") { + return new Response(JSON.stringify({ ok: true, restarting: true }), { + status: 200, + }); + } + healthChecks += 1; + if (healthChecks === 1) throw new TypeError("daemon is down"); + return new Response(null, { status: 200 }); + }) as typeof fetch; + + const restarting = triggerRestart(); + await vi.runAllTimersAsync(); + await restarting; + + expect(healthChecks).toBe(2); + expect(fakeWindow.location.href).toMatch(/^\/\?_t=\d+$/); + }); +}); diff --git a/apps/memos-local-plugin/viewer/src/stores/restart.ts b/apps/memos-local-plugin/viewer/src/stores/restart.ts index 974a9e222..6d089f784 100644 --- a/apps/memos-local-plugin/viewer/src/stores/restart.ts +++ b/apps/memos-local-plugin/viewer/src/stores/restart.ts @@ -4,10 +4,9 @@ * OpenClaw can be restarted from the viewer because the plugin lives * inside the gateway process and launchd brings it back. * - * Hermes is different: the viewer daemon is intentionally long-lived, - * so restart means "terminate the active `hermes chat` process" while - * keeping this Memory Viewer online. The user can relaunch Hermes and - * it will reconnect to the existing viewer service. + * Hermes has separate chat and viewer bridge processes. The backend + * terminates the active chat and replaces the viewer daemon so both + * processes load the saved config. */ import { signal } from "@preact/signals"; import { api } from "../api/client"; @@ -58,8 +57,6 @@ async function pollHealthUntilUp(maxAttempts = 60): Promise { /** * Quick health check for destructive clear-data only. - * Do not use this for Hermes config saves: those must keep the current - * viewer daemon online and terminate only the active `hermes chat`. */ async function quickPollUp(maxAttempts = 30): Promise { for (let i = 0; i < maxAttempts; i++) { @@ -75,9 +72,8 @@ async function quickPollUp(maxAttempts = 30): Promise { } /** - * Config saved. OpenClaw gets an in-place gateway restart. Hermes keeps - * the viewer online and asks the backend to terminate the active chat - * process; once that request returns, reload the same viewer page. + * Config saved. OpenClaw gets an in-place gateway restart. Hermes + * replaces its viewer daemon and terminates the active chat process. * * Do not add a passive "settings saved" toast/card here. The restart * affordance is intentionally blocking for both agents so the operator @@ -88,13 +84,19 @@ export async function triggerRestart(): Promise { if (!isOpenClaw()) { try { await api.post("/api/v1/admin/restart"); - await new Promise((r) => setTimeout(r, 500)); - window.location.href = - window.location.pathname + "?_t=" + Date.now(); } catch { restartState.value = { phase: "restartFailed" }; throw new Error("restart failed"); } + + const ok = await pollHealthUntilUp(60); + if (ok) { + window.location.href = + window.location.pathname + "?_t=" + Date.now(); + } else { + restartState.value = { phase: "restartFailed" }; + throw new Error("restart did not complete"); + } return; } From b8d8f88f6e0e65c1d78bf77969dfed5adcda27b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=81=E5=9C=A8=E5=90=B5=E7=9D=80=E5=90=83=E7=B3=96?= Date: Tue, 28 Jul 2026 19:05:18 +0800 Subject: [PATCH 04/18] fix(plugin): share Hermes bridge across providers --- .../adapters/hermes/README.md | 73 ++- .../hermes/memos_provider/__init__.py | 342 ++++++---- .../memos_provider/shared_bridge_runtime.py | 606 ++++++++++++++++++ .../tests/python/test_bridge_client.py | 10 +- .../python/test_hermes_provider_pipeline.py | 12 + .../python/test_shared_bridge_runtime.py | 371 +++++++++++ 6 files changed, 1263 insertions(+), 151 deletions(-) create mode 100644 apps/memos-local-plugin/adapters/hermes/memos_provider/shared_bridge_runtime.py create mode 100644 apps/memos-local-plugin/tests/python/test_shared_bridge_runtime.py diff --git a/apps/memos-local-plugin/adapters/hermes/README.md b/apps/memos-local-plugin/adapters/hermes/README.md index 1bd98cd0a..9f05755c9 100644 --- a/apps/memos-local-plugin/adapters/hermes/README.md +++ b/apps/memos-local-plugin/adapters/hermes/README.md @@ -10,18 +10,20 @@ Hermes is a Python-based agent. The core of `memos-local-plugin` is written in TypeScript and runs on Node.js. This adapter bridges the two: ``` -┌──────────────────────────┐ stdio JSON-RPC 2.0 ┌─────────────────────┐ -│ Hermes Python process │ ─────────────────────────────▶ │ node bridge.cts │ -│ │ │ │ -│ MemTensorProvider ──────┼───── turn.start / turn.end ───▶│ MemoryCore (core/) │ -│ │◀──── events.notify ───────────│ │ -│ │◀──── logs.forward ────────────│ │ -└──────────────────────────┘ └─────────────────────┘ +┌────────────────────────────────────┐ stdio JSON-RPC 2.0 ┌─────────────────────┐ +│ Hermes Python process │ │ one Node bridge │ +│ │ │ │ +│ Provider(session A) ─┐ │ │ MemoryCore (core/) │ +│ Provider(session B) ─┼─ shared ────┼───────────────────────────▶│ │ +│ Provider(session C) ─┘ runtime │◀── events / host LLM RPC ─│ │ +└────────────────────────────────────┘ └─────────────────────┘ ``` -The Python side is **stateless** — every `MemoryProvider` method is a thin -proxy that translates to a JSON-RPC call. All algorithm logic (L1/L2/L3, -skills, retrieval, feedback, decision repair) lives in the shared TS core. +Each Python provider owns its logical session/episode/turn state. The +process-scoped `SharedBridgeRuntime` owns the physical Node subprocess, +keepalive, reconnect generation, and host callback dispatch. All algorithm +logic (L1/L2/L3, skills, retrieval, feedback, decision repair) remains in the +shared TypeScript core. ## Protocol surface @@ -29,16 +31,14 @@ The adapter calls the following methods on the bridge: | Hermes hook | JSON-RPC method | Purpose | | ---------------------------- | ---------------------- | ------------------------------------------------- | -| `initialize(session_id)` | `session.open`, | Open a session + default episode in core. | -| | `episode.open` | | +| `initialize(session_id)` | `session.open` | Open one logical session in the shared core. | | `prefetch(query)` | `turn.start` | Retrieve context for injection before model call. | -| `sync_turn(user, assistant)` | `turn.end` (deferred) | Persist a completed turn — flushed async. | -| `handle_tool_call("memory_*")`| `memory.search`, | Explicit memory tools exposed to the model. | +| `sync_turn(user, assistant)` | `turn.end` | Persist a completed turn synchronously. | +| `handle_tool_call("memory_*")` | `memory.search`, | Explicit memory tools exposed to the model. | | | `memory.timeline` | | | `submit_feedback(...)` | `feedback.submit` | Record explicit user feedback. | -| `on_session_end` | `episode.close`, | Close the session and flush pending data. | -| | `session.close` | | -| `shutdown` | (transport close) | Kill the subprocess cleanly. | +| `on_session_end` | `session.close` | Close only this logical session. | +| `shutdown` | (lease release) | Release this provider without killing the bridge. | ## File layout @@ -49,6 +49,7 @@ adapters/hermes/ └── memos_provider/ ├── __init__.py # MemTensorProvider — the MemoryProvider impl ├── bridge_client.py # JSON-RPC 2.0 stdio client + thread-safe dispatch + ├── shared_bridge_runtime.py # Process-scoped ownership, reconnect, hook routing └── daemon_manager.py # Spawn lifecycle + probe for Node availability ``` @@ -60,11 +61,25 @@ startup. If Node.js is unavailable the provider reports provider. No deployment artifacts from this adapter are required on machines that can't run Node. -Otherwise the provider spawns `node --experimental-strip-types -bridge.cts --agent=hermes` as a subprocess during `initialize()` and -communicates over its stdin/stdout pipes. The subprocess exits when the -provider's stdin closes (on `shutdown()`), yielding clean lifecycle -semantics. +Otherwise the first provider in a Hermes process starts the packaged bridge +(`dist/bridge.mjs` when available) with `--agent=hermes --no-viewer`. +Subsequent providers acquire leases on the same bridge. The bridge stays alive +until process exit, while each provider independently opens and closes its +logical `sessionId`. + +The runtime uses one keepalive loop and a generation-checked reconnect lock. +After a bridge restart, a provider lazily reopens its own logical session before +issuing its next request. Long-running in-flight capture/retrieval calls suppress +keepalive probes so a health timeout cannot kill healthy work. + +For emergency rollback, set: + +```sh +MEMOS_HERMES_BRIDGE_MODE=legacy +``` + +This restores the old per-provider lifecycle and should be used only for +diagnosis because it is unsafe when several Hermes sessions coexist. ## Why a subprocess instead of a long-lived daemon? @@ -76,9 +91,14 @@ That approach required: - authentication between Python and Node, - and duplicate log pipelines. -The stdio model gets each of those for free from the OS process model. -Hermes' session lifetime is already bounded, so the cost of one -`node`-spawn per Hermes session is negligible in practice. +The stdio model gets each of those for free from the OS process model. Sharing +one stdio bridge inside each Hermes process also prevents separate providers +from repeatedly replacing each other's subprocesses while preserving the +reverse `host.llm.complete` callback. + +This is process-scoped, not machine-scoped. A future daemon transport can share +one MemoryCore across multiple Hermes OS processes, but it must retain a +bidirectional callback channel before replacing stdio. ## Testing @@ -86,4 +106,5 @@ Python unit tests live under `apps/memos-local-plugin/tests/python/`. They run against a mocked bridge (no Node subprocess) to exercise the Hermes-side state machine. Integration tests that exercise the full stack boot a real bridge subprocess; see -`tests/python/integration/test_hermes_roundtrip.py`. +`tests/python/test_shared_bridge_runtime.py` and the Hermes protocol/persistence +tests under `tests/unit/adapters/`. diff --git a/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py b/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py index d0c9bdb36..16a36c0fd 100644 --- a/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py +++ b/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py @@ -27,20 +27,20 @@ | Hermes hook | Our action | | -------------------- | --------------------------------------------- | -| ``initialize`` | spawn bridge; open session + episode | +| ``initialize`` | acquire shared bridge; open logical session | | ``on_turn_start`` | record turn count; stash message | | ``prefetch`` | ``turn.start`` RPC → Tier 1+2+3 retrieval | -| ``queue_prefetch`` | background thread: prefetch + flush pending | -| ``sync_turn`` | queue a deferred ``turn.end`` RPC | -| ``on_session_end`` | flush pending + close episode + close session | +| ``queue_prefetch`` | no-op; real prefetch runs before the turn | +| ``sync_turn`` | persist a synchronous ``turn.end`` RPC | +| ``on_session_end`` | close this logical session | | ``on_pre_compress`` | extract a short memory summary | | ``on_delegation`` | record a subagent outcome as a trace | | ``get_tool_schemas`` | expose memory, skill, and environment tools | | ``handle_tool_call`` | dispatch to MemOS JSON-RPC tool methods | -| ``shutdown`` | close bridge | +| ``shutdown`` | release provider lease | -Threading: all JSON-RPC calls are synchronous. ``queue_prefetch`` runs on -a daemon thread the provider owns. +Threading: all JSON-RPC calls are synchronous. One process-scoped runtime +owns bridge keepalive and reconnect; providers retain per-session state. """ from __future__ import annotations @@ -71,6 +71,11 @@ ensure_viewer_daemon, kill_zombie_bridges, ) +from shared_bridge_runtime import ( + HERMES_HOOK_DISPATCHER, + SHARED_BRIDGE_REGISTRY, + SharedBridgeLease, +) try: # pragma: no cover — host-provided base class, absent in unit tests @@ -110,6 +115,43 @@ def _read_plugin_version() -> str: _COMPRESSION_CONTEXT_MAX_CHARS = 6_000 +def _shared_bridge_enabled() -> bool: + """Use the safe process-scoped bridge unless an operator opts out.""" + mode = os.environ.get("MEMOS_HERMES_BRIDGE_MODE", "shared").strip().lower() + return mode not in {"legacy", "per_provider", "disabled", "false", "0"} + + +def _resolved_memos_runtime_home() -> Path: + """Mirror the Node resolver closely enough to isolate distinct databases.""" + memos_home = os.environ.get("MEMOS_HOME", "").strip() + if memos_home: + return Path(memos_home).expanduser().resolve() + config_file = os.environ.get("MEMOS_CONFIG_FILE", "").strip() + if config_file: + return Path(config_file).expanduser().resolve().parent + return (Path.home() / ".hermes" / "memos-plugin").resolve() + + +def _shared_bridge_runtime_key() -> tuple[str, ...]: + plugin_root = _PLUGIN_DIR.parents[2].resolve() + runtime_home = _resolved_memos_runtime_home() + node_binary = os.environ.get("MEMOS_NODE_BINARY", "").strip() + return (str(plugin_root), str(runtime_home), node_binary, "hermes", "stdio") + + +def _prepare_shared_bridge() -> None: + """Run process-level bridge/viewer preparation exactly once per spawn.""" + ensure_bridge_running() + with contextlib.suppress(Exception): + zombies = kill_zombie_bridges() + if zombies: + logger.info("MemOS: killed %d zombie bridge(s)", zombies) + try: + ensure_viewer_daemon() + except Exception as err: + logger.warning("MemOS: viewer daemon check failed — %s", err) + + def _long_rpc_timeout_default() -> float: """Resolve the timeout used for long-running JSON-RPC calls. @@ -291,8 +333,11 @@ class MemTensorProvider(MemoryProvider): """ def __init__(self) -> None: - self._bridge: MemosBridgeClient | None = None + self._bridge: MemosBridgeClient | SharedBridgeLease | None = None + self._shared_bridge = _shared_bridge_enabled() + self._bridge_generation = 0 self._reconnect_lock = threading.Lock() + self._session_open_lock = threading.Lock() self._session_id: str = "" self._episode_id: str = "" self._hermes_home: str = "" @@ -325,6 +370,8 @@ def __init__(self) -> None: self._hook_registered = False self._bridge_keepalive_stop = threading.Event() self._bridge_keepalive_thread: threading.Thread | None = None + self._session_close_thread: threading.Thread | None = None + self._session_close_requested_for = "" # Hermes runs background memory/skill reviewers by forking an agent and # appending a synthetic user turn. That turn is instruction plumbing, # not a human utterance, so it must not become a MemOS trace. @@ -354,107 +401,77 @@ def initialize(self, session_id: str, **kwargs: Any) -> None: # type: ignore[ov them so the bridge can resolve the right `~/.hermes/memos-plugin/` and log the originating channel. - We open the session here but NOT the episode — episode creation - is deferred to ``_ensure_episode()`` (called from the first - ``on_turn_start``), so the actual user message can be passed as - the episode's initial text instead of a generic placeholder. - - Idempotency (issue #1910): the host occasionally re-enters - ``initialize()`` on the same provider instance (plugin reload, - session restart). Without the close-before-spawn guard below the - previous ``MemosBridgeClient`` would be replaced by reference - only, orphaning the previous Node subprocess and accumulating - a fresh ``bridge.cjs`` per turn. + Shared mode (the default) acquires a lightweight lease on one + process-scoped Node bridge. Each provider still owns its logical + session/episode state. ``MEMOS_HERMES_BRIDGE_MODE=legacy`` retains + the old per-provider bridge as a one-release rollback path. """ - # Hermes can call `initialize()` multiple times across a single - # parent process (e.g. on reconnect or a new session). Each call - # spawns a fresh `MemosBridgeClient` (and therefore a new - # `--no-viewer` Node subprocess); if we simply overwrite - # `self._bridge` we leak the old subprocess — its parent stays - # alive so `_reap_stale_headless_bridges_locked` never collects - # it. Close the previous bridge first, mirroring the safe pattern - # already used by `_reconnect_bridge()`. See #1927. previous_bridge = self._bridge if previous_bridge is not None: old_pid = getattr(previous_bridge, "pid", "?") logger.info( - "MemOS: closing previous bridge (pid=%s) before re-init", + "MemOS: releasing previous bridge handle (pid=%s) before re-init", old_pid, ) + self.on_session_end([]) + close_thread = self._session_close_thread + if close_thread is not None and close_thread.is_alive(): + close_thread.join(timeout=5.5) with contextlib.suppress(Exception): previous_bridge.close() self._bridge = None + self._bridge_generation = 0 self._session_id = session_id or self._session_id + self._session_close_requested_for = "" + self._bridge_keepalive_stop.clear() with self._state_lock: self._last_injected_context = "" self._active_turn_key = "" self._hermes_home = str(kwargs.get("hermes_home") or "") self._platform = str(kwargs.get("platform") or "cli") self._agent_identity = str(kwargs.get("agent_identity") or "hermes") - if self._bridge is not None: - prev_pid = getattr(self._bridge, "pid", "?") - logger.info( - "MemOS: initialize() invoked while bridge already exists; " - "closing previous bridge (pid=%s) before respawn", - prev_pid, - ) - old_bridge = self._bridge - self._bridge = None - with contextlib.suppress(Exception): - old_bridge.close() - try: - ensure_bridge_running() - except Exception as err: - logger.warning("MemOS: failed to start bridge — %s", err) - return + self._shared_bridge = _shared_bridge_enabled() - # Kill zombie bridges from previous sessions before deciding - # how to connect. + new_bridge: MemosBridgeClient | SharedBridgeLease | None = None try: - zombies = kill_zombie_bridges() - if zombies: - logger.info("MemOS: killed %d zombie bridge(s)", zombies) - except Exception: - pass - - # NOTE: An HTTP bridge path used to live here that connected to a - # running viewer daemon over HTTP instead of spawning a stdio - # subprocess. It depended on ``MemosHttpClient`` which was never - # committed — the class was referenced by name only. Issue #2096 - # reverts the half-merged HTTP feature; the stdio path below is - # the sole connection mechanism until the HTTP client lands as a - # complete change. - - if self._bridge is None: - try: - ensure_viewer_daemon() - except Exception as err: - logger.warning("MemOS: viewer daemon check failed — %s", err) - new_bridge: MemosBridgeClient | None = None - try: + if self._shared_bridge: + new_bridge = SHARED_BRIDGE_REGISTRY.acquire( + _shared_bridge_runtime_key(), + client_factory=lambda: MemosBridgeClient(), + before_spawn=_prepare_shared_bridge, + host_handlers={ + "host.llm.complete": self._handle_host_llm_complete, + }, + ) + else: + _prepare_shared_bridge() new_bridge = MemosBridgeClient() - # Register the fallback LLM handler BEFORE we open the - # session so it is available the very first time the - # plugin's facade asks for help (e.g. on the first - # `turn.start` retrieval call). new_bridge.register_host_handler( "host.llm.complete", self._handle_host_llm_complete, ) - self._bridge = new_bridge - self._open_session(session_id, timeout=60.0) - logger.info( - "MemOS: bridge ready (stdio) session=%s platform=%s (episode deferred)", - self._session_id, - self._platform, - ) - except Exception as err: - logger.warning("MemOS: bridge init failed — %s", err) - if new_bridge is not None: - with contextlib.suppress(Exception): - new_bridge.close() - self._bridge = None + self._bridge = new_bridge + self._open_session(session_id, timeout=60.0) + mode = "shared" if self._shared_bridge else "legacy" + runtime_id = getattr(new_bridge, "runtime_id", "per-provider") + logger.info( + "MemOS: bridge ready mode=%s runtime=%s generation=%d pid=%s " + "session=%s platform=%s (episode deferred)", + mode, + runtime_id, + self._bridge_generation, + getattr(new_bridge, "pid", "?"), + self._session_id, + self._platform, + ) + except Exception as err: + logger.warning("MemOS: bridge init failed — %s", err) + if new_bridge is not None: + with contextlib.suppress(Exception): + new_bridge.close() + self._bridge = None + self._bridge_generation = 0 # Register a Hermes plugin hook to capture tool calls as they # happen. The `post_tool_call` hook fires after every tool # dispatch (write_file, terminal, search_files, etc.) with the @@ -534,23 +551,15 @@ def _record_namespace(self) -> dict[str, Any]: return ns def _register_tool_call_hook(self) -> None: - if self._hook_registered: - return try: from hermes_cli.plugins import ( get_plugin_manager, # pyright: ignore[reportMissingImports] ) mgr = get_plugin_manager() - mgr._hooks.setdefault("post_tool_call", []).append(self._on_post_tool_call) - mgr._hooks.setdefault("post_llm_call", []).append(self._on_post_llm_call) - mgr._hooks.setdefault("transform_tool_result", []).append( - self._on_transform_tool_result - ) + HERMES_HOOK_DISPATCHER.bind(mgr, self) self._hook_registered = True - logger.debug( - "MemOS: registered post_tool_call + post_llm_call + transform_tool_result hooks" - ) + logger.debug("MemOS: bound provider session=%s to shared hooks", self._session_id) except Exception as err: logger.debug("MemOS: could not register tool hook — %s", err) @@ -1136,7 +1145,7 @@ def on_delegation( "hookKwargs": kwargs, "namespace": namespace, } - self._bridge.request( + self._bridge_request( "subagent.record", { "agent": "hermes", @@ -1641,14 +1650,29 @@ def on_session_end(self, messages: list[dict[str, Any]]) -> None: # type: ignor # the asyncio event loop (gateway/run.py calls us synchronously from # _handle_reset_command) and causes Discord heartbeat timeouts when the # bridge is unresponsive. 5 s timeout keeps it bounded. - _bridge = self._bridge _sid = self._session_id + if not _sid or self._session_close_requested_for == _sid: + return + self._session_close_requested_for = _sid def _close() -> None: - with contextlib.suppress(Exception): - _bridge.request("session.close", {"sessionId": _sid}, timeout=5.0) + try: + self._bridge_request( + "session.close", + {"sessionId": _sid}, + timeout=5.0, + ) + except Exception as err: + if self._session_close_requested_for == _sid: + self._session_close_requested_for = "" + logger.debug("MemOS: session.close failed session=%s — %s", _sid, err) - threading.Thread(target=_close, daemon=True).start() + self._session_close_thread = threading.Thread( + target=_close, + daemon=True, + name="memos-session-close", + ) + self._session_close_thread.start() def __del__(self) -> None: # Safety net — if shutdown() was never called (e.g. caller forgot, @@ -1667,6 +1691,8 @@ def shutdown(self) -> None: # type: ignore[override] with self._state_lock: self._last_injected_context = "" self._active_turn_key = "" + HERMES_HOOK_DISPATCHER.unbind(self) + self._hook_registered = False self._bridge_keepalive_stop.set() if self._bridge_keepalive_thread and self._bridge_keepalive_thread.is_alive(): self._bridge_keepalive_thread.join( @@ -1675,12 +1701,20 @@ def shutdown(self) -> None: # type: ignore[override] if self._prefetch_thread and self._prefetch_thread.is_alive(): self._prefetch_thread.join(timeout=5.0) if self._bridge: + self.on_session_end([]) + close_thread = self._session_close_thread + if close_thread is not None and close_thread.is_alive(): + close_thread.join(timeout=5.5) pid = getattr(self._bridge, "pid", "?") - logger.info("MemOS: shutting down bridge (pid=%s)", pid) + action = ( + "releasing shared bridge lease" if self._shared_bridge else "shutting down bridge" + ) + logger.info("MemOS: %s (pid=%s session=%s)", action, pid, self._session_id) with contextlib.suppress(Exception): self._bridge.close() self._bridge = None - logger.info("MemOS: bridge shutdown complete (pid=%s)", pid) + self._bridge_generation = 0 + logger.info("MemOS: provider bridge shutdown complete (pid=%s)", pid) # ─── Host LLM bridge (fallback for plugin-side model failures) ──────── @@ -1852,11 +1886,41 @@ def _host_runtime_context(self) -> dict[str, str]: self._last_host_runtime = dict(out) return out + def _bridge_request( + self, + method: str, + params: Any = None, + *, + timeout: float | None = None, + ensure_session: bool = True, + ) -> dict[str, Any]: + bridge = self._bridge + if bridge is None: + raise BridgeError("transport_closed", "bridge is not connected") + if ensure_session and isinstance(bridge, SharedBridgeLease): + generation = bridge.generation + if generation != self._bridge_generation: + with self._session_open_lock: + if bridge.generation != self._bridge_generation: + logger.info( + "MemOS: reopening logical session after shared bridge generation " + "change runtime=%s old_generation=%d new_generation=%d session=%s", + bridge.runtime_id, + self._bridge_generation, + bridge.generation, + self._session_id, + ) + self._open_session(self._session_id, timeout=30.0) + if timeout is None: + return bridge.request(method, params) + return bridge.request(method, params, timeout=timeout) + def _open_session(self, session_id: str = "", *, timeout: float = 30.0) -> None: - assert self._bridge is not None + bridge = self._bridge + assert bridge is not None requested_session = session_id or self._session_id or "" host_runtime = self._host_runtime_context() - resp = self._bridge.request( + resp = bridge.request( "session.open", { "agent": "hermes", @@ -1874,6 +1938,10 @@ def _open_session(self, session_id: str = "", *, timeout: float = 30.0) -> None: timeout=timeout, ) self._session_id = resp.get("sessionId") or requested_session + if isinstance(bridge, SharedBridgeLease): + self._bridge_generation = bridge.generation + else: + self._bridge_generation += 1 def _bridge_request_with_retry( self, @@ -1896,9 +1964,7 @@ def _bridge_request_with_retry( """ assert self._bridge is not None try: - if timeout is None: - return self._bridge.request(method, params) - return self._bridge.request(method, params, timeout=timeout) + return self._bridge_request(method, params, timeout=timeout) except BridgeError as err: if not self._is_transport_closed(err): raise @@ -1909,9 +1975,7 @@ def _bridge_request_with_retry( ) self._reconnect_bridge(self._session_id, timeout=30.0) assert self._bridge is not None - if timeout is None: - return self._bridge.request(method, params) - return self._bridge.request(method, params, timeout=timeout) + return self._bridge_request(method, params, timeout=timeout) def _is_transport_closed(self, err: Exception) -> bool: if isinstance(err, BridgeError) and err.code == "transport_closed": @@ -1962,6 +2026,36 @@ def _reconnect_bridge(self, session_id: str = "", *, timeout: float = 30.0) -> N logger.debug("MemOS: skipping reconnect during shutdown (after lock)") return + if self._shared_bridge: + bridge = self._bridge + if bridge is None: + bridge = SHARED_BRIDGE_REGISTRY.acquire( + _shared_bridge_runtime_key(), + client_factory=lambda: MemosBridgeClient(), + before_spawn=_prepare_shared_bridge, + host_handlers={ + "host.llm.complete": self._handle_host_llm_complete, + }, + ) + self._bridge = bridge + if not isinstance(bridge, SharedBridgeLease): + raise BridgeError( + "internal", + "shared bridge mode has a non-shared bridge handle", + ) + expected_generation = self._bridge_generation or bridge.generation + bridge.reconnect(expected_generation=expected_generation) + self._open_session(session_id, timeout=timeout) + logger.info( + "MemOS: shared bridge session recovered runtime=%s generation=%d " + "pid=%s session=%s", + bridge.runtime_id, + bridge.generation, + bridge.pid, + self._session_id, + ) + return + old_bridge = self._bridge old_pid = getattr(old_bridge, "pid", None) if old_bridge else None @@ -1971,15 +2065,7 @@ def _reconnect_bridge(self, session_id: str = "", *, timeout: float = 30.0) -> N old_bridge.close() logger.info("MemOS: old bridge closed (pid=%s)", old_pid) - ensure_bridge_running() - # NOTE: HTTP bridge reconnect path was removed alongside issue - # #2096. See ``initialize`` for the rationale. Reconnect always - # spawns a fresh stdio bridge. - - try: - ensure_viewer_daemon() - except Exception as err: - logger.warning("MemOS: viewer daemon check failed during reconnect — %s", err) + _prepare_shared_bridge() new_bridge: MemosBridgeClient | None = None try: new_bridge = MemosBridgeClient() @@ -2016,6 +2102,9 @@ def _ensure_bridge(self, session_id: str = "", *, timeout: float = 30.0) -> bool return False def _start_bridge_keepalive(self) -> None: + if self._shared_bridge: + # SharedBridgeRuntime owns the only keepalive/reconnect loop. + return if self._bridge_keepalive_thread and self._bridge_keepalive_thread.is_alive(): return self._bridge_keepalive_stop.clear() @@ -2075,16 +2164,25 @@ def _turn_start(self, query: str, *, session_id: str = "") -> str: } if turn_key: payload["turnKey"] = turn_key - resp = self._bridge.request( + resp = self._bridge_request_with_retry( "turn.start", payload, timeout=_LONG_RPC_TIMEOUT, ) + response_query = (resp or {}).get("query") or {} + response_session = str(response_query.get("sessionId") or "") + requested_session = str(payload["sessionId"]) + if response_session and response_session != requested_session: + raise BridgeError( + "session_mismatch", + "turn.start returned a different session " + f"(requested={requested_session}, returned={response_session})", + ) # Stash the real episode id the pipeline auto-created (V7 # §0.1 may have boundary-cut the previous episode and started # a new one). `on_session_end` uses it to close the right # episode — see the "Episode tracking" comment block above. - new_eid = ((resp or {}).get("query") or {}).get("episodeId") or "" + new_eid = response_query.get("episodeId") or "" if new_eid and new_eid != self._episode_id: self._episode_id = new_eid logger.debug("MemOS: stashed episode %s from turn.start", new_eid) @@ -2125,7 +2223,7 @@ def _turn_end( } if agent_thinking: payload["agentThinking"] = agent_thinking - result = self._bridge.request("turn.end", payload, timeout=_LONG_RPC_TIMEOUT) + result = self._bridge_request("turn.end", payload, timeout=_LONG_RPC_TIMEOUT) # Capture the trace ID for feedback submission if result and isinstance(result, dict): trace_ids = result.get("traceIds", []) @@ -2187,7 +2285,7 @@ def _submit_verifier_feedback( } if trace_id: payload["traceId"] = trace_id - self._bridge.request("feedback.submit", payload, timeout=75.0) + self._bridge_request("feedback.submit", payload, timeout=75.0) return True diff --git a/apps/memos-local-plugin/adapters/hermes/memos_provider/shared_bridge_runtime.py b/apps/memos-local-plugin/adapters/hermes/memos_provider/shared_bridge_runtime.py new file mode 100644 index 000000000..a024a1505 --- /dev/null +++ b/apps/memos-local-plugin/adapters/hermes/memos_provider/shared_bridge_runtime.py @@ -0,0 +1,606 @@ +"""Process-scoped bridge ownership and Hermes hook routing. + +Hermes creates one memory provider per cached agent/session. The MemOS Node +bridge, however, owns process-wide resources (MemoryCore, SQLite connections, +model clients, and background workers) that are designed to multiplex logical +``sessionId`` values. This module keeps exactly one physical bridge per +runtime/data-home key while handing providers lightweight leases. +""" + +from __future__ import annotations + +import atexit +import contextlib +import logging +import threading +import time +import weakref + +from typing import TYPE_CHECKING, Any + +from bridge_client import BridgeError, MemosBridgeClient + + +if TYPE_CHECKING: + from collections.abc import Callable, Hashable + + +logger = logging.getLogger(__name__) + +_TRANSPORT_ERROR_MARKERS = ("broken pipe", "bridge closed", "transport_closed") + + +def _is_transport_closed(err: Exception) -> bool: + if isinstance(err, BridgeError) and err.code == "transport_closed": + return True + return any(marker in str(err).lower() for marker in _TRANSPORT_ERROR_MARKERS) + + +class _CallableRef: + """Weak reference wrapper that also supports non-weak-referenceable callables.""" + + def __init__(self, callback: Callable[..., Any]) -> None: + self._strong: Callable[..., Any] | None = None + try: + if getattr(callback, "__self__", None) is not None: + self._ref: Callable[[], Callable[..., Any] | None] = weakref.WeakMethod(callback) + else: + self._ref = weakref.ref(callback) + except TypeError: + self._strong = callback + self._ref = lambda: self._strong + + def get(self) -> Callable[..., Any] | None: + return self._ref() + + +class SharedBridgeRuntime: + """Own one physical bridge and safely multiplex provider leases.""" + + def __init__( + self, + key: tuple[Hashable, ...], + *, + client_factory: Callable[[], MemosBridgeClient], + before_spawn: Callable[[], None] | None = None, + keepalive_interval: float = 10.0, + keepalive_timeout: float = 10.0, + ) -> None: + self.key = key + self.runtime_id = f"hermes-{abs(hash(key)):x}" + self._client_factory = client_factory + self._before_spawn = before_spawn + self._keepalive_interval = max(0.1, keepalive_interval) + self._keepalive_timeout = max(0.1, keepalive_timeout) + + self._state_lock = threading.RLock() + self._reconnect_lock = threading.Lock() + self._client: MemosBridgeClient | None = None + self._generation = 0 + self._leases: set[int] = set() + self._next_lease_id = 1 + self._in_flight = 0 + self._host_handlers: dict[str, dict[int, _CallableRef]] = {} + self._registered_host_methods: set[str] = set() + + self._failure_count = 0 + self._next_reconnect_at = 0.0 + self._last_reconnect_error: Exception | None = None + self._keepalive_failures = 0 + self._stop = threading.Event() + self._keepalive_thread: threading.Thread | None = None + + @property + def generation(self) -> int: + with self._state_lock: + return self._generation + + @property + def pid(self) -> int: + with self._state_lock: + client = self._client + return int(getattr(client, "pid", 0) or 0) if client is not None else 0 + + def acquire( + self, + *, + host_handlers: dict[str, Callable[[dict[str, Any]], Any]] | None = None, + ) -> SharedBridgeLease: + with self._state_lock: + lease_id = self._next_lease_id + self._next_lease_id += 1 + self._leases.add(lease_id) + lease = SharedBridgeLease(self, lease_id) + client = self._client + newly_registered_methods: list[str] = [] + for method, handler in (host_handlers or {}).items(): + self._host_handlers.setdefault(method, {})[lease_id] = _CallableRef(handler) + if method not in self._registered_host_methods: + self._registered_host_methods.add(method) + newly_registered_methods.append(method) + allow_backoff = self._generation > 0 or self._failure_count > 0 + should_start_keepalive = ( + self._keepalive_thread is None or not self._keepalive_thread.is_alive() + ) + # Existing clients need newly introduced reverse handlers immediately. + # New clients install all known dispatchers before their first health + # request, preventing startup-time host RPC from blocking the reader. + try: + if client is not None: + for method in newly_registered_methods: + client.register_host_handler(method, self._host_dispatcher(method)) + self.ensure_client(allow_backoff=allow_backoff) + except Exception: + self.release(lease_id) + with self._state_lock: + for method in newly_registered_methods: + if not self._host_handlers.get(method): + self._registered_host_methods.discard(method) + raise + if should_start_keepalive: + self._start_keepalive() + logger.debug( + "MemOS: acquired shared bridge lease runtime=%s generation=%d pid=%s leases=%d", + self.runtime_id, + self.generation, + self.pid, + self.lease_count, + ) + return lease + + @property + def lease_count(self) -> int: + with self._state_lock: + return len(self._leases) + + def ensure_client(self, *, allow_backoff: bool = True) -> None: + with self._state_lock: + if self._client is not None: + return + generation = self._generation + self.reconnect(expected_generation=generation, allow_backoff=allow_backoff) + + def request( + self, + lease_id: int, + method: str, + params: Any = None, + *, + timeout: float = 30.0, + ) -> dict[str, Any]: + with self._state_lock: + if lease_id not in self._leases: + raise BridgeError("transport_closed", "shared bridge lease is closed") + client = self._client + self._in_flight += 1 + try: + if client is None: + self.ensure_client(allow_backoff=True) + with self._state_lock: + client = self._client + if client is None: # pragma: no cover - defensive after ensure_client + raise BridgeError("transport_closed", "shared bridge is unavailable") + return client.request(method, params, timeout=timeout) + finally: + with self._state_lock: + self._in_flight = max(0, self._in_flight - 1) + + def reconnect( + self, + *, + expected_generation: int | None = None, + allow_backoff: bool = True, + ) -> int: + """Replace the bridge once; stale concurrent reconnects reuse the winner.""" + with self._reconnect_lock: + with self._state_lock: + if ( + expected_generation is not None + and self._client is not None + and self._generation != expected_generation + ): + return self._generation + if allow_backoff and time.monotonic() < self._next_reconnect_at: + err = self._last_reconnect_error + message = "shared bridge reconnect is backing off" + if err is not None: + message = f"{message}: {err}" + raise BridgeError("transport_closed", message) + old_client = self._client + old_pid = int(getattr(old_client, "pid", 0) or 0) if old_client else 0 + self._client = None + + if old_client is not None: + with contextlib.suppress(Exception): + old_client.close() + + new_client: MemosBridgeClient | None = None + try: + if self._before_spawn is not None: + self._before_spawn() + new_client = self._client_factory() + self._install_host_dispatchers(new_client) + new_client.request("core.health", {}, timeout=self._keepalive_timeout) + except Exception as err: + if new_client is not None: + with contextlib.suppress(Exception): + new_client.close() + self._record_reconnect_failure(err) + raise + + with self._state_lock: + self._client = new_client + self._generation += 1 + self._failure_count = 0 + self._next_reconnect_at = 0.0 + self._last_reconnect_error = None + self._keepalive_failures = 0 + generation = self._generation + lease_count = len(self._leases) + logger.info( + "MemOS: shared bridge ready runtime=%s generation=%d pid=%s old_pid=%s leases=%d", + self.runtime_id, + generation, + self.pid, + old_pid or "-", + lease_count, + ) + return generation + + def register_host_handler( + self, + lease_id: int, + method: str, + handler: Callable[[dict[str, Any]], Any], + ) -> None: + with self._state_lock: + if lease_id not in self._leases: + return + handlers = self._host_handlers.setdefault(method, {}) + handlers[lease_id] = _CallableRef(handler) + client = self._client + needs_registration = method not in self._registered_host_methods + if needs_registration: + self._registered_host_methods.add(method) + if needs_registration and client is not None: + client.register_host_handler(method, self._host_dispatcher(method)) + + def release(self, lease_id: int) -> None: + with self._state_lock: + self._leases.discard(lease_id) + for handlers in self._host_handlers.values(): + handlers.pop(lease_id, None) + leases = len(self._leases) + logger.debug( + "MemOS: released shared bridge lease runtime=%s generation=%d pid=%s leases=%d", + self.runtime_id, + self.generation, + self.pid, + leases, + ) + + def status(self) -> dict[str, Any]: + with self._state_lock: + return { + "runtimeId": self.runtime_id, + "generation": self._generation, + "pid": self.pid, + "leases": len(self._leases), + "inFlight": self._in_flight, + "reconnectFailures": self._failure_count, + "keepaliveFailures": self._keepalive_failures, + } + + def close(self) -> None: + self._stop.set() + thread = self._keepalive_thread + if thread is not None and thread.is_alive(): + thread.join(timeout=min(self._keepalive_timeout + 1.0, 12.0)) + with self._state_lock: + client = self._client + self._client = None + self._leases.clear() + self._host_handlers.clear() + if client is not None: + with contextlib.suppress(Exception): + client.close() + + def _record_reconnect_failure(self, err: Exception) -> None: + with self._state_lock: + self._failure_count += 1 + delay = min(30.0, float(2 ** min(self._failure_count - 1, 5))) + self._next_reconnect_at = time.monotonic() + delay + self._last_reconnect_error = err + failure_count = self._failure_count + logger.warning( + "MemOS: shared bridge reconnect failed runtime=%s failures=%d retry_in=%.1fs — %s", + self.runtime_id, + failure_count, + delay, + err, + ) + + def _host_dispatcher(self, method: str) -> Callable[[dict[str, Any]], Any]: + def dispatch(params: dict[str, Any]) -> Any: + with self._state_lock: + handlers = self._host_handlers.get(method, {}) + candidates = list(reversed(handlers.items())) + stale: list[int] = [] + for lease_id, callback_ref in candidates: + callback = callback_ref.get() + if callback is not None: + return callback(params) + stale.append(lease_id) + if stale: + with self._state_lock: + current = self._host_handlers.get(method, {}) + for lease_id in stale: + current.pop(lease_id, None) + raise BridgeError("host_unavailable", f"no active host handler for {method}") + + return dispatch + + def _install_host_dispatchers(self, client: MemosBridgeClient) -> None: + with self._state_lock: + methods = list(self._registered_host_methods) + for method in methods: + client.register_host_handler(method, self._host_dispatcher(method)) + + def _start_keepalive(self) -> None: + with self._state_lock: + if self._keepalive_thread is not None and self._keepalive_thread.is_alive(): + return + self._stop.clear() + thread = threading.Thread( + target=self._keepalive_loop, + daemon=True, + name=f"memos-shared-bridge-keepalive-{self.runtime_id}", + ) + self._keepalive_thread = thread + thread.start() + + def _keepalive_loop(self) -> None: + while not self._stop.wait(self._keepalive_interval): + with self._state_lock: + client = self._client + generation = self._generation + in_flight = self._in_flight + # A capture/retrieval call may legitimately spend up to the + # provider's long-RPC timeout in an embedding or LLM request. + # Do not let a concurrent health timeout kill healthy in-flight + # work for every session. + if in_flight: + continue + if client is None: + with contextlib.suppress(Exception): + self.reconnect(expected_generation=generation) + continue + try: + client.request("core.health", {}, timeout=self._keepalive_timeout) + with self._state_lock: + self._keepalive_failures = 0 + except Exception as err: + with self._state_lock: + self._keepalive_failures += 1 + failures = self._keepalive_failures + should_reconnect = _is_transport_closed(err) or ( + isinstance(err, BridgeError) and err.code == "timeout" and failures >= 2 + ) + if not should_reconnect: + logger.debug( + "MemOS: shared bridge keepalive failed runtime=%s failures=%d — %s", + self.runtime_id, + failures, + err, + ) + continue + logger.warning( + "MemOS: shared bridge keepalive reconnecting runtime=%s " + "generation=%d failures=%d — %s", + self.runtime_id, + generation, + failures, + err, + ) + with contextlib.suppress(Exception): + self.reconnect(expected_generation=generation) + + +class SharedBridgeLease: + """Provider-facing handle that never owns or closes the physical bridge.""" + + def __init__(self, runtime: SharedBridgeRuntime, lease_id: int) -> None: + self._runtime = runtime + self._lease_id = lease_id + self._closed = False + + @property + def pid(self) -> int: + return self._runtime.pid + + @property + def generation(self) -> int: + return self._runtime.generation + + @property + def runtime_id(self) -> str: + return self._runtime.runtime_id + + def request( + self, + method: str, + params: Any = None, + *, + timeout: float = 30.0, + ) -> dict[str, Any]: + if self._closed: + raise BridgeError("transport_closed", "shared bridge lease is closed") + return self._runtime.request(self._lease_id, method, params, timeout=timeout) + + def reconnect(self, *, expected_generation: int | None = None) -> int: + if self._closed: + raise BridgeError("transport_closed", "shared bridge lease is closed") + return self._runtime.reconnect(expected_generation=expected_generation) + + def register_host_handler( + self, + method: str, + handler: Callable[[dict[str, Any]], Any], + ) -> None: + self._runtime.register_host_handler(self._lease_id, method, handler) + + def status(self) -> dict[str, Any]: + return self._runtime.status() + + def close(self) -> None: + if self._closed: + return + self._closed = True + self._runtime.release(self._lease_id) + + +class SharedBridgeRuntimeRegistry: + """Registry of shared runtimes, isolated by resolved MemOS data home.""" + + def __init__( + self, + *, + keepalive_interval: float = 10.0, + keepalive_timeout: float = 10.0, + ) -> None: + self._lock = threading.Lock() + self._runtimes: dict[tuple[Hashable, ...], SharedBridgeRuntime] = {} + self._keepalive_interval = keepalive_interval + self._keepalive_timeout = keepalive_timeout + + def acquire( + self, + key: tuple[Hashable, ...], + *, + client_factory: Callable[[], MemosBridgeClient], + before_spawn: Callable[[], None] | None = None, + host_handlers: dict[str, Callable[[dict[str, Any]], Any]] | None = None, + ) -> SharedBridgeLease: + with self._lock: + runtime = self._runtimes.get(key) + if runtime is None: + runtime = SharedBridgeRuntime( + key, + client_factory=client_factory, + before_spawn=before_spawn, + keepalive_interval=self._keepalive_interval, + keepalive_timeout=self._keepalive_timeout, + ) + self._runtimes[key] = runtime + # Keep a failed runtime entry so its exponential backoff applies to + # later providers. Removing it here would let each new Hermes session + # create a fresh runtime and bypass the circuit breaker. + return runtime.acquire(host_handlers=host_handlers) + + def close_all(self) -> None: + with self._lock: + runtimes = list(self._runtimes.values()) + self._runtimes.clear() + for runtime in runtimes: + runtime.close() + + def status(self) -> list[dict[str, Any]]: + with self._lock: + runtimes = list(self._runtimes.values()) + return [runtime.status() for runtime in runtimes] + + +class HermesHookDispatcher: + """Register one global Hermes hook set and route events by session.""" + + def __init__(self) -> None: + self._lock = threading.RLock() + self._providers: dict[int, weakref.ReferenceType[Any]] = {} + self._registered_managers: set[int] = set() + self._last_ambiguous_warning = 0.0 + + def bind(self, manager: Any, provider: Any) -> None: + with self._lock: + self._providers[id(provider)] = weakref.ref(provider) + manager_id = id(manager) + if manager_id in self._registered_managers: + return + manager._hooks.setdefault("post_tool_call", []).append(self._post_tool_call) + manager._hooks.setdefault("post_llm_call", []).append(self._post_llm_call) + manager._hooks.setdefault("transform_tool_result", []).append( + self._transform_tool_result + ) + self._registered_managers.add(manager_id) + logger.debug("MemOS: registered one shared Hermes hook dispatcher") + + def unbind(self, provider: Any) -> None: + with self._lock: + self._providers.pop(id(provider), None) + + def _live_providers(self) -> list[Any]: + with self._lock: + items = list(self._providers.items()) + live: list[Any] = [] + stale: list[int] = [] + for provider_id, provider_ref in items: + provider = provider_ref() + if provider is None: + stale.append(provider_id) + else: + live.append(provider) + if stale: + with self._lock: + for provider_id in stale: + self._providers.pop(provider_id, None) + return live + + def _select(self, kwargs: dict[str, Any]) -> Any | None: + session_id = str(kwargs.get("session_id") or kwargs.get("sessionId") or "") + providers = self._live_providers() + if session_id: + matches = [p for p in providers if str(getattr(p, "_session_id", "")) == session_id] + return matches[-1] if matches else None + if len(providers) == 1: + return providers[0] + if len(providers) > 1: + now = time.monotonic() + with self._lock: + if now - self._last_ambiguous_warning >= 60.0: + self._last_ambiguous_warning = now + logger.warning( + "MemOS: ignored sessionless Hermes hook with %d active providers; " + "event was not broadcast", + len(providers), + ) + return None + + def _post_tool_call(self, **kwargs: Any) -> None: + provider = self._select(kwargs) + if provider is not None: + provider._on_post_tool_call(**kwargs) + + def _post_llm_call(self, **kwargs: Any) -> None: + provider = self._select(kwargs) + if provider is not None: + provider._on_post_llm_call(**kwargs) + + def _transform_tool_result(self, **kwargs: Any) -> str | None: + provider = self._select(kwargs) + if provider is None: + return None + return provider._on_transform_tool_result(**kwargs) + + +SHARED_BRIDGE_REGISTRY = SharedBridgeRuntimeRegistry() +HERMES_HOOK_DISPATCHER = HermesHookDispatcher() +atexit.register(SHARED_BRIDGE_REGISTRY.close_all) + + +__all__ = [ + "HERMES_HOOK_DISPATCHER", + "SHARED_BRIDGE_REGISTRY", + "HermesHookDispatcher", + "SharedBridgeLease", + "SharedBridgeRuntime", + "SharedBridgeRuntimeRegistry", +] diff --git a/apps/memos-local-plugin/tests/python/test_bridge_client.py b/apps/memos-local-plugin/tests/python/test_bridge_client.py index 3322de7ed..673961c3f 100644 --- a/apps/memos-local-plugin/tests/python/test_bridge_client.py +++ b/apps/memos-local-plugin/tests/python/test_bridge_client.py @@ -29,10 +29,10 @@ if str(_p) not in sys.path: sys.path.insert(0, str(_p)) -import bridge_client as bridge_client_mod # noqa: E402 -import daemon_manager as daemon_manager_mod # noqa: E402 +import bridge_client as bridge_client_mod +import daemon_manager as daemon_manager_mod -from bridge_client import BridgeError, MemosBridgeClient # noqa: E402 +from bridge_client import BridgeError, MemosBridgeClient class FakePopen: @@ -473,8 +473,10 @@ def setUp(self) -> None: import memos_provider self._provider_mod = memos_provider + self._provider_mod.SHARED_BRIDGE_REGISTRY.close_all() self._patches = [ + patch.dict("os.environ", {"MEMOS_HERMES_BRIDGE_MODE": "legacy"}), patch("memos_provider.ensure_bridge_running", return_value=True), patch("memos_provider.ensure_viewer_daemon", return_value=True), ] @@ -482,6 +484,7 @@ def setUp(self) -> None: p.start() def tearDown(self) -> None: + self._provider_mod.SHARED_BRIDGE_REGISTRY.close_all() for p in self._patches: p.stop() @@ -677,6 +680,7 @@ def test_on_turn_start_stashes_message(self) -> None: def test_on_delegation_is_noop_without_bridge(self) -> None: p = self._provider_mod.MemTensorProvider() + p._bridge_keepalive_stop.set() p.on_delegation("run tests", "all green") # must not raise def test_on_pre_compress_without_bridge_returns_empty(self) -> None: diff --git a/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py b/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py index 887ccafbe..445af36c6 100644 --- a/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py +++ b/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py @@ -68,6 +68,18 @@ def request(self, method: str, params: dict | None = None, **_kwargs: object) -> class HermesProviderPipelineTests(unittest.TestCase): + def setUp(self) -> None: + memos_provider.SHARED_BRIDGE_REGISTRY.close_all() + self._mode_patch = patch.dict( + "os.environ", + {"MEMOS_HERMES_BRIDGE_MODE": "legacy"}, + ) + self._mode_patch.start() + + def tearDown(self) -> None: + memos_provider.SHARED_BRIDGE_REGISTRY.close_all() + self._mode_patch.stop() + def test_plugin_version_matches_package_version(self) -> None: package_json = _ADAPTER_ROOT.parent.parent / "package.json" package_version = json.loads(package_json.read_text(encoding="utf-8"))["version"] diff --git a/apps/memos-local-plugin/tests/python/test_shared_bridge_runtime.py b/apps/memos-local-plugin/tests/python/test_shared_bridge_runtime.py new file mode 100644 index 000000000..b74be2f54 --- /dev/null +++ b/apps/memos-local-plugin/tests/python/test_shared_bridge_runtime.py @@ -0,0 +1,371 @@ +"""Concurrency and lifecycle tests for the Hermes shared bridge runtime.""" + +from __future__ import annotations + +import sys +import threading +import unittest + +from pathlib import Path +from unittest.mock import patch + + +_ADAPTER_ROOT = Path(__file__).resolve().parent.parent.parent / "adapters" / "hermes" +_PLUGIN_DIR = _ADAPTER_ROOT / "memos_provider" +for _p in (_ADAPTER_ROOT, _PLUGIN_DIR): + if str(_p) not in sys.path: + sys.path.insert(0, str(_p)) + +import memos_provider + +from bridge_client import BridgeError +from shared_bridge_runtime import ( + HermesHookDispatcher, + SharedBridgeRuntimeRegistry, +) + + +class FakeBridge: + _next_pid = 4100 + + def __init__(self) -> None: + type(self)._next_pid += 1 + self.pid = type(self)._next_pid + self.closed = False + self.calls: list[tuple[str, dict]] = [] + self.handlers: dict[str, object] = {} + + def request(self, method: str, params: dict | None = None, **_kwargs: object) -> dict: + if self.closed: + raise BridgeError("transport_closed", "bridge client is closed") + payload = params or {} + self.calls.append((method, payload)) + if method == "core.health": + return {"ok": True} + if method == "session.open": + return {"sessionId": payload.get("sessionId")} + if method == "turn.start": + return { + "query": { + "sessionId": payload.get("sessionId"), + "episodeId": f"episode:{payload.get('sessionId')}", + }, + "injectedContext": "shared memory", + } + if method == "turn.end": + return {"traceIds": [f"trace:{payload.get('sessionId')}"]} + return {} + + def register_host_handler(self, method: str, handler: object) -> None: + self.handlers[method] = handler + + def close(self) -> None: + self.closed = True + + +class FakeManager: + def __init__(self) -> None: + self._hooks: dict[str, list[object]] = {} + + +class FakeProvider: + def __init__(self, session_id: str) -> None: + self._session_id = session_id + self.tool_calls = 0 + self.llm_calls = 0 + self.transforms = 0 + + def _on_post_tool_call(self, **_kwargs: object) -> None: + self.tool_calls += 1 + + def _on_post_llm_call(self, **_kwargs: object) -> None: + self.llm_calls += 1 + + def _on_transform_tool_result(self, **_kwargs: object) -> str: + self.transforms += 1 + return f"handled:{self._session_id}" + + +class SharedBridgeRuntimeTests(unittest.TestCase): + def setUp(self) -> None: + self.registry = SharedBridgeRuntimeRegistry( + keepalive_interval=3600.0, + keepalive_timeout=0.1, + ) + + def tearDown(self) -> None: + self.registry.close_all() + + def test_two_leases_share_one_bridge_and_release_does_not_close_it(self) -> None: + created: list[FakeBridge] = [] + + def factory() -> FakeBridge: + bridge = FakeBridge() + created.append(bridge) + return bridge + + first = self.registry.acquire(("shared-home",), client_factory=factory) + second = self.registry.acquire(("shared-home",), client_factory=factory) + + self.assertEqual(len(created), 1) + self.assertEqual(first.pid, second.pid) + + first.close() + self.assertFalse(created[0].closed) + self.assertEqual(second.request("core.health"), {"ok": True}) + + second.close() + self.assertFalse(created[0].closed) + + def test_concurrent_reconnect_creates_one_replacement(self) -> None: + created: list[FakeBridge] = [] + + def factory() -> FakeBridge: + bridge = FakeBridge() + created.append(bridge) + return bridge + + first = self.registry.acquire(("shared-home",), client_factory=factory) + second = self.registry.acquire(("shared-home",), client_factory=factory) + generation = first.generation + barrier = threading.Barrier(3) + + def reconnect(lease) -> None: + barrier.wait() + lease.reconnect(expected_generation=generation) + + threads = [ + threading.Thread(target=reconnect, args=(first,)), + threading.Thread(target=reconnect, args=(second,)), + ] + for thread in threads: + thread.start() + barrier.wait() + for thread in threads: + thread.join(timeout=2.0) + + self.assertEqual(len(created), 2) + self.assertTrue(created[0].closed) + self.assertEqual(first.generation, generation + 1) + self.assertEqual(first.pid, second.pid) + + def test_host_handler_is_registered_once_per_bridge_generation(self) -> None: + bridge = FakeBridge() + lease_a = self.registry.acquire(("shared-home",), client_factory=lambda: bridge) + lease_b = self.registry.acquire(("shared-home",), client_factory=lambda: bridge) + + handler_a = lambda _params: {"content": "a"} + handler_b = lambda _params: {"content": "b"} + lease_a.register_host_handler("host.llm.complete", handler_a) + lease_b.register_host_handler("host.llm.complete", handler_b) + + self.assertEqual(list(bridge.handlers), ["host.llm.complete"]) + dispatcher = bridge.handlers["host.llm.complete"] + self.assertEqual(dispatcher({}), {"content": "b"}) # type: ignore[operator] + + lease_b.close() + self.assertEqual(dispatcher({}), {"content": "a"}) # type: ignore[operator] + + def test_initial_host_handler_is_installed_before_first_health_check(self) -> None: + class StartupBridge(FakeBridge): + def request( + self, + method: str, + params: dict | None = None, + **kwargs: object, + ) -> dict: + if method == "core.health": + self_test.assertIn("host.llm.complete", self.handlers) + return super().request(method, params, **kwargs) + + self_test = self + handler = lambda _params: {"content": "ready"} + bridge = StartupBridge() + lease = self.registry.acquire( + ("shared-home",), + client_factory=lambda: bridge, + host_handlers={"host.llm.complete": handler}, + ) + + dispatcher = bridge.handlers["host.llm.complete"] + self.assertEqual(dispatcher({}), {"content": "ready"}) # type: ignore[operator] + lease.close() + + def test_runtime_keys_isolate_distinct_data_homes(self) -> None: + created: list[FakeBridge] = [] + + def factory() -> FakeBridge: + bridge = FakeBridge() + created.append(bridge) + return bridge + + first = self.registry.acquire(("home-a",), client_factory=factory) + second = self.registry.acquire(("home-b",), client_factory=factory) + + self.assertEqual(len(created), 2) + self.assertNotEqual(first.pid, second.pid) + + def test_failed_new_acquire_does_not_revoke_existing_leases(self) -> None: + attempts = 0 + + def factory() -> FakeBridge: + nonlocal attempts + attempts += 1 + if attempts > 1: + raise RuntimeError("bridge spawn failed") + return FakeBridge() + + lease = self.registry.acquire(("shared-home",), client_factory=factory) + with self.assertRaises(RuntimeError): + lease.reconnect(expected_generation=lease.generation) + with self.assertRaises(BridgeError) as blocked: + self.registry.acquire(("shared-home",), client_factory=factory) + + self.assertIn("backing off", str(blocked.exception)) + self.assertEqual(lease.status()["leases"], 1) + + +class HermesHookDispatcherTests(unittest.TestCase): + def test_registers_one_global_hook_set_and_routes_by_session(self) -> None: + manager = FakeManager() + dispatcher = HermesHookDispatcher() + first = FakeProvider("session-a") + second = FakeProvider("session-b") + + dispatcher.bind(manager, first) + dispatcher.bind(manager, second) + dispatcher.bind(manager, second) + + self.assertEqual(len(manager._hooks["post_tool_call"]), 1) + self.assertEqual(len(manager._hooks["post_llm_call"]), 1) + self.assertEqual(len(manager._hooks["transform_tool_result"]), 1) + + manager._hooks["post_tool_call"][0](session_id="session-b") # type: ignore[operator] + self.assertEqual(first.tool_calls, 0) + self.assertEqual(second.tool_calls, 1) + + result = manager._hooks["transform_tool_result"][0](session_id="session-a") # type: ignore[operator] + self.assertEqual(result, "handled:session-a") + self.assertEqual(first.transforms, 1) + self.assertEqual(second.transforms, 0) + + def test_ambiguous_sessionless_event_is_not_broadcast(self) -> None: + manager = FakeManager() + dispatcher = HermesHookDispatcher() + first = FakeProvider("session-a") + second = FakeProvider("session-b") + dispatcher.bind(manager, first) + dispatcher.bind(manager, second) + + manager._hooks["post_tool_call"][0](session_id="") # type: ignore[operator] + + self.assertEqual(first.tool_calls, 0) + self.assertEqual(second.tool_calls, 0) + + def test_unbind_removes_provider_without_unregistering_global_hooks(self) -> None: + manager = FakeManager() + dispatcher = HermesHookDispatcher() + provider = FakeProvider("session-a") + dispatcher.bind(manager, provider) + + dispatcher.unbind(provider) + manager._hooks["post_llm_call"][0](session_id="session-a") # type: ignore[operator] + + self.assertEqual(provider.llm_calls, 0) + self.assertEqual(len(manager._hooks["post_llm_call"]), 1) + + +class SharedProviderIntegrationTests(unittest.TestCase): + def setUp(self) -> None: + memos_provider.SHARED_BRIDGE_REGISTRY.close_all() + self._mode_patch = patch.dict( + "os.environ", + { + "MEMOS_HERMES_BRIDGE_MODE": "shared", + "MEMOS_HOME": "/tmp/memos-hermes-shared-runtime-test", + }, + ) + self._mode_patch.start() + + def tearDown(self) -> None: + memos_provider.SHARED_BRIDGE_REGISTRY.close_all() + self._mode_patch.stop() + + def test_two_providers_share_physical_bridge_and_keep_sessions_isolated(self) -> None: + created: list[FakeBridge] = [] + + def factory() -> FakeBridge: + bridge = FakeBridge() + created.append(bridge) + return bridge + + with ( + patch("memos_provider.MemosBridgeClient", side_effect=factory), + patch("memos_provider.ensure_bridge_running", return_value=True), + patch("memos_provider.ensure_viewer_daemon", return_value=True), + patch("memos_provider.kill_zombie_bridges", return_value=0), + ): + first = memos_provider.MemTensorProvider() + second = memos_provider.MemTensorProvider() + first.initialize("session-a") + second.initialize("session-b") + + self.assertEqual(len(created), 1) + self.assertEqual(first._bridge.pid, second._bridge.pid) + + first.prefetch("remember A") + second.prefetch("remember B") + first.sync_turn("remember A", "stored A") + second.sync_turn("remember B", "stored B") + + first.shutdown() + self.assertFalse(created[0].closed) + self.assertIn("Recalled Memories", second.prefetch("still alive")) + second.shutdown() + self.assertFalse(created[0].closed) + + session_ids = [ + payload.get("sessionId") + for method, payload in created[0].calls + if method in {"session.open", "turn.start", "turn.end", "session.close"} + ] + self.assertIn("session-a", session_ids) + self.assertIn("session-b", session_ids) + + def test_provider_reopens_its_session_after_another_lease_reconnects(self) -> None: + created: list[FakeBridge] = [] + + def factory() -> FakeBridge: + bridge = FakeBridge() + created.append(bridge) + return bridge + + with ( + patch("memos_provider.MemosBridgeClient", side_effect=factory), + patch("memos_provider.ensure_bridge_running", return_value=True), + patch("memos_provider.ensure_viewer_daemon", return_value=True), + patch("memos_provider.kill_zombie_bridges", return_value=0), + ): + first = memos_provider.MemTensorProvider() + second = memos_provider.MemTensorProvider() + first.initialize("session-a") + second.initialize("session-b") + + old_generation = first._bridge.generation + first._bridge.reconnect(expected_generation=old_generation) + self.assertEqual(len(created), 2) + + second.handle_tool_call("memos_search", {"query": "after reconnect"}) + methods = [method for method, _payload in created[1].calls] + session_open_index = methods.index("session.open") + search_index = methods.index("memory.search") + self.assertLess(session_open_index, search_index) + reopened = created[1].calls[session_open_index][1] + self.assertEqual(reopened["sessionId"], "session-b") + + first.shutdown() + second.shutdown() + + +if __name__ == "__main__": + unittest.main() From 956381a93802dfc4097c793054933253a5eddebf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=81=E5=9C=A8=E5=90=B5=E7=9D=80=E5=90=83=E7=B3=96?= Date: Tue, 28 Jul 2026 19:38:48 +0800 Subject: [PATCH 05/18] fix(plugin): isolate Hermes bridges by data home --- .../hermes/memos_provider/__init__.py | 126 +++++++++--- .../hermes/memos_provider/bridge_client.py | 85 ++++++-- apps/memos-local-plugin/bridge.cts | 10 +- .../tests/python/test_bridge_client.py | 59 +++++- .../python/test_hermes_provider_pipeline.py | 11 +- .../python/test_shared_bridge_runtime.py | 189 +++++++++++++++++- 6 files changed, 432 insertions(+), 48 deletions(-) diff --git a/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py b/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py index 16a36c0fd..e663b87cb 100644 --- a/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py +++ b/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py @@ -132,20 +132,42 @@ def _resolved_memos_runtime_home() -> Path: return (Path.home() / ".hermes" / "memos-plugin").resolve() -def _shared_bridge_runtime_key() -> tuple[str, ...]: - plugin_root = _PLUGIN_DIR.parents[2].resolve() - runtime_home = _resolved_memos_runtime_home() - node_binary = os.environ.get("MEMOS_NODE_BINARY", "").strip() - return (str(plugin_root), str(runtime_home), node_binary, "hermes", "stdio") +def _memos_runtime_env_snapshot(runtime_home: Path | None = None) -> dict[str, str]: + """Freeze the environment inputs that select a MemOS data home.""" + memos_home = os.environ.get("MEMOS_HOME", "").strip() + if memos_home: + resolved_home = runtime_home or Path(memos_home).expanduser().resolve() + return {"MEMOS_HOME": str(resolved_home)} + config_file = os.environ.get("MEMOS_CONFIG_FILE", "").strip() + if config_file: + return { + "MEMOS_HOME": "", + "MEMOS_CONFIG_FILE": str(Path(config_file).expanduser().resolve()), + } + return { + "MEMOS_HOME": "", + "MEMOS_CONFIG_FILE": "", + "HOME": os.environ.get("HOME", "").strip() or str(Path.home()), + } -def _prepare_shared_bridge() -> None: - """Run process-level bridge/viewer preparation exactly once per spawn.""" +def _shared_bridge_runtime_key(runtime_home: Path | None = None) -> tuple[str, ...]: + """Use the data home as the authoritative shared-runtime boundary.""" + resolved_home = runtime_home or _resolved_memos_runtime_home() + return (str(resolved_home), "hermes", "stdio") + + +def _prepare_shared_bridge(*, cleanup_legacy_zombies: bool = False) -> None: + """Prepare bridge/viewer state without crossing data-home boundaries.""" ensure_bridge_running() - with contextlib.suppress(Exception): - zombies = kill_zombie_bridges() - if zombies: - logger.info("MemOS: killed %d zombie bridge(s)", zombies) + if cleanup_legacy_zombies: + # The legacy scanner cannot distinguish data homes. Shared runtimes + # instead rely on the Python scoped singleton and the CJS scoped PID + # guard so one healthy home is never reaped while another starts. + with contextlib.suppress(Exception): + zombies = kill_zombie_bridges() + if zombies: + logger.info("MemOS: killed %d zombie bridge(s)", zombies) try: ensure_viewer_daemon() except Exception as err: @@ -338,6 +360,8 @@ def __init__(self) -> None: self._bridge_generation = 0 self._reconnect_lock = threading.Lock() self._session_open_lock = threading.Lock() + self._runtime_home: Path | None = None + self._runtime_env: dict[str, str] = {} self._session_id: str = "" self._episode_id: str = "" self._hermes_home: str = "" @@ -432,21 +456,33 @@ def initialize(self, session_id: str, **kwargs: Any) -> None: # type: ignore[ov self._platform = str(kwargs.get("platform") or "cli") self._agent_identity = str(kwargs.get("agent_identity") or "hermes") self._shared_bridge = _shared_bridge_enabled() + self._runtime_home = _resolved_memos_runtime_home() + self._runtime_env = _memos_runtime_env_snapshot(self._runtime_home) new_bridge: MemosBridgeClient | SharedBridgeLease | None = None try: + runtime_home = self._runtime_home + runtime_env = dict(self._runtime_env) if self._shared_bridge: new_bridge = SHARED_BRIDGE_REGISTRY.acquire( - _shared_bridge_runtime_key(), - client_factory=lambda: MemosBridgeClient(), + _shared_bridge_runtime_key(runtime_home), + client_factory=lambda home=str(runtime_home), env=runtime_env: ( + MemosBridgeClient( + runtime_home=home, + extra_env=env, + ) + ), before_spawn=_prepare_shared_bridge, host_handlers={ "host.llm.complete": self._handle_host_llm_complete, }, ) else: - _prepare_shared_bridge() - new_bridge = MemosBridgeClient() + _prepare_shared_bridge(cleanup_legacy_zombies=True) + new_bridge = MemosBridgeClient( + runtime_home=str(runtime_home), + extra_env=runtime_env, + ) new_bridge.register_host_handler( "host.llm.complete", self._handle_host_llm_complete, @@ -2028,10 +2064,21 @@ def _reconnect_bridge(self, session_id: str = "", *, timeout: float = 30.0) -> N if self._shared_bridge: bridge = self._bridge + acquired_here = bridge is None if bridge is None: + if self._runtime_home is None: + self._runtime_home = _resolved_memos_runtime_home() + self._runtime_env = _memos_runtime_env_snapshot(self._runtime_home) + runtime_home = self._runtime_home + runtime_env = dict(self._runtime_env) bridge = SHARED_BRIDGE_REGISTRY.acquire( - _shared_bridge_runtime_key(), - client_factory=lambda: MemosBridgeClient(), + _shared_bridge_runtime_key(runtime_home), + client_factory=lambda home=str(runtime_home), env=runtime_env: ( + MemosBridgeClient( + runtime_home=home, + extra_env=env, + ) + ), before_spawn=_prepare_shared_bridge, host_handlers={ "host.llm.complete": self._handle_host_llm_complete, @@ -2043,9 +2090,31 @@ def _reconnect_bridge(self, session_id: str = "", *, timeout: float = 30.0) -> N "internal", "shared bridge mode has a non-shared bridge handle", ) - expected_generation = self._bridge_generation or bridge.generation - bridge.reconnect(expected_generation=expected_generation) - self._open_session(session_id, timeout=timeout) + try: + if acquired_here: + # acquire() already ensures and health-checks the shared + # client. Open this logical session directly; restarting + # here would disrupt every healthy provider using it. + acquired_generation = bridge.generation + try: + self._open_session(session_id, timeout=timeout) + except Exception as err: + if not self._is_transport_closed(err): + raise + bridge.reconnect(expected_generation=acquired_generation) + self._open_session(session_id, timeout=timeout) + else: + expected_generation = self._bridge_generation or bridge.generation + bridge.reconnect(expected_generation=expected_generation) + self._open_session(session_id, timeout=timeout) + except Exception: + if acquired_here: + with contextlib.suppress(Exception): + bridge.close() + if self._bridge is bridge: + self._bridge = None + self._bridge_generation = 0 + raise logger.info( "MemOS: shared bridge session recovered runtime=%s generation=%d " "pid=%s session=%s", @@ -2065,11 +2134,21 @@ def _reconnect_bridge(self, session_id: str = "", *, timeout: float = 30.0) -> N old_bridge.close() logger.info("MemOS: old bridge closed (pid=%s)", old_pid) - _prepare_shared_bridge() + _prepare_shared_bridge(cleanup_legacy_zombies=True) new_bridge: MemosBridgeClient | None = None try: - new_bridge = MemosBridgeClient() - logger.info("MemOS: new bridge created (pid=%s)", getattr(new_bridge, "pid", "?")) + runtime_home = self._runtime_home or _resolved_memos_runtime_home() + runtime_env = dict( + self._runtime_env or _memos_runtime_env_snapshot(runtime_home) + ) + new_bridge = MemosBridgeClient( + runtime_home=str(runtime_home), + extra_env=runtime_env, + ) + logger.info( + "MemOS: new bridge created (pid=%s)", + getattr(new_bridge, "pid", "?"), + ) new_bridge.register_host_handler( "host.llm.complete", @@ -2098,7 +2177,6 @@ def _ensure_bridge(self, session_id: str = "", *, timeout: float = 30.0) -> bool return True except Exception as err: logger.warning("MemOS: bridge reconnect failed — %s", err) - self._bridge = None return False def _start_bridge_keepalive(self) -> None: diff --git a/apps/memos-local-plugin/adapters/hermes/memos_provider/bridge_client.py b/apps/memos-local-plugin/adapters/hermes/memos_provider/bridge_client.py index bdc650667..841f4b687 100644 --- a/apps/memos-local-plugin/adapters/hermes/memos_provider/bridge_client.py +++ b/apps/memos-local-plugin/adapters/hermes/memos_provider/bridge_client.py @@ -13,6 +13,7 @@ from __future__ import annotations import contextlib +import hashlib import json import logging import os @@ -33,18 +34,47 @@ HOST_HANDLER_WAIT_SECONDS = 5.0 # ─── Module-level singleton tracker ───────────────────────────────────── -# Each entry maps a ``(agent, no_viewer)`` key to the most-recent active -# ``MemosBridgeClient`` for that slot. When a new client is constructed -# for an existing key, the previous client is closed synchronously so the -# Node-side ``bridge.cjs`` subprocess does not leak. +# Each entry maps an ``(agent, no_viewer, runtime_home)`` key to the +# most-recent active ``MemosBridgeClient`` for that slot. When a new client +# is constructed for an existing key, the previous client is closed +# synchronously so the Node-side ``bridge.cjs`` subprocess does not leak. # # This is the Python-side guard against issue #1910 (bridge process leak: # every turn spawns new bridge.cjs). Defence in depth on the Node side # lives in ``bridge.cts`` via ``bridge-stdio.pid``. -_ACTIVE_CLIENTS: dict[tuple[str, bool], MemosBridgeClient] = {} +_ACTIVE_CLIENTS: dict[tuple[str, bool, str], MemosBridgeClient] = {} _ACTIVE_CLIENTS_LOCK = threading.Lock() +def _expanded_path(value: str, env: dict[str, str]) -> Path: + """Resolve a path using the child process' HOME rather than global state.""" + home = env.get("HOME", "").strip() or str(Path.home()) + if value == "~": + value = home + elif value.startswith("~/") or value.startswith("~\\"): + value = str(Path(home) / value[2:]) + return Path(value).resolve() + + +def _resolved_runtime_home(agent: str, env: dict[str, str]) -> Path: + """Mirror the Node home resolver for singleton/process ownership.""" + memos_home = env.get("MEMOS_HOME", "").strip() + if memos_home: + return _expanded_path(memos_home, env) + config_file = env.get("MEMOS_CONFIG_FILE", "").strip() + if config_file: + return _expanded_path(config_file, env).parent + agent_home = ".hermes" if agent == "hermes" else f".{agent}" + default_home = Path(env.get("HOME", "") or Path.home()) / agent_home / "memos-plugin" + return _expanded_path(str(default_home), env) + + +def _runtime_scope_token(agent: str, runtime_home: Path) -> str: + """Return a stable, path-private token safe for use in a PID filename.""" + raw = f"{agent}\0{runtime_home}".encode("utf-8") + return hashlib.sha256(raw).hexdigest()[:24] + + def _installed_node_binary(plugin_root: Path) -> str | None: marker = plugin_root / ".memos-node-bin" try: @@ -112,6 +142,7 @@ def __init__( agent: str = "hermes", no_viewer: bool = True, extra_env: dict[str, str] | None = None, + runtime_home: str | None = None, ) -> None: self._lock = threading.Lock() self._next_id = 1 @@ -140,6 +171,21 @@ def __init__( script_path = Path(bridge_path) if bridge_path else _bridge_script(plugin_root) script = str(script_path) env = {**os.environ, **(extra_env or {})} + resolved_runtime_home = ( + _expanded_path(runtime_home, env) + if runtime_home + else _resolved_runtime_home(agent, env) + ) + if runtime_home and not { + "MEMOS_HOME", + "MEMOS_CONFIG_FILE", + }.intersection(extra_env or {}): + # An explicit home is both an ownership scope and the child + # runtime location. Callers that deliberately snapshot either + # selector in extra_env retain that more specific configuration. + env["MEMOS_HOME"] = str(resolved_runtime_home) + env.pop("MEMOS_CONFIG_FILE", None) + runtime_scope = _runtime_scope_token(agent, resolved_runtime_home) # Prefer the compiled JavaScript bridge — the new pure ESM # ``dist/bridge.mjs`` (issue #1736) or the legacy ``dist/bridge.cjs`` @@ -150,7 +196,11 @@ def __init__( # so use tsx's real JS entrypoint whenever we have to launch the # source entry through a specific Node. tsx_cli = plugin_root / "node_modules" / "tsx" / "dist" / "cli.mjs" - bridge_args = [script, f"--agent={agent}"] + bridge_args = [ + script, + f"--agent={agent}", + f"--runtime-scope={runtime_scope}", + ] if no_viewer: bridge_args.append("--no-viewer") if script_path.suffix in (".mjs", ".cjs"): @@ -190,13 +240,14 @@ def __init__( self._stderr_reader.start() # Singleton tracking (issue #1910). Register ourselves as the - # active client for ``(agent, no_viewer)`` and reap any previous - # holder synchronously so its subprocess does not leak. The reap - # happens AFTER our reader threads are running, so the previous - # client's ``close()`` (which closes stdin and waits for exit) - # cannot interfere with our own startup. + # active client for ``(agent, no_viewer, runtime_home)`` and reap + # any previous holder synchronously so its subprocess does not leak. + # The reap happens AFTER our reader threads are running, so the + # previous client's ``close()`` (which closes stdin and waits for + # exit) cannot interfere with our own startup. self._singleton_agent = agent self._singleton_no_viewer = bool(no_viewer) + self._singleton_runtime_home = str(resolved_runtime_home) previous = self._register_active() if previous is not None and previous is not self: prev_pid = getattr(previous, "pid", "?") @@ -210,7 +261,11 @@ def __init__( def _register_active(self) -> MemosBridgeClient | None: """Register self as the active singleton; return the displaced client.""" - key = (self._singleton_agent, self._singleton_no_viewer) + key = ( + self._singleton_agent, + self._singleton_no_viewer, + self._singleton_runtime_home, + ) with _ACTIVE_CLIENTS_LOCK: previous = _ACTIVE_CLIENTS.get(key) _ACTIVE_CLIENTS[key] = self @@ -218,7 +273,11 @@ def _register_active(self) -> MemosBridgeClient | None: def _unregister_active(self) -> None: """Remove self from the active registry if we are still the current entry.""" - key = (self._singleton_agent, self._singleton_no_viewer) + key = ( + self._singleton_agent, + self._singleton_no_viewer, + self._singleton_runtime_home, + ) with _ACTIVE_CLIENTS_LOCK: if _ACTIVE_CLIENTS.get(key) is self: _ACTIVE_CLIENTS.pop(key, None) diff --git a/apps/memos-local-plugin/bridge.cts b/apps/memos-local-plugin/bridge.cts index e26203836..86bcdd07a 100644 --- a/apps/memos-local-plugin/bridge.cts +++ b/apps/memos-local-plugin/bridge.cts @@ -52,6 +52,7 @@ interface BridgeArgs { tcpPort?: number; agent: "openclaw" | "hermes"; home?: string; + runtimeScope?: string; } type BridgeStatus = "connected" | "reconnecting" | "disconnected" | "unknown"; @@ -72,6 +73,10 @@ function parseArgs(argv: readonly string[]): BridgeArgs { else if (raw === "--agent=hermes") args.agent = "hermes"; else if (raw === "--agent=openclaw") args.agent = "openclaw"; else if (raw.startsWith("--home=")) args.home = raw.slice(7); + else if (raw.startsWith("--runtime-scope=")) { + const candidate = raw.slice(16).toLowerCase(); + if (/^[a-f0-9]{16,64}$/.test(candidate)) args.runtimeScope = candidate; + } } return args; } @@ -158,7 +163,10 @@ async function main(): Promise { // ─── Singleton: kill previous bridge that owns the viewer port ─── const pidPath = pidFilePath(args.agent); - const stdioPidPath = pidFilePath(args.agent, STDIO_PID_FILENAME); + const stdioPidFilename = args.runtimeScope + ? `bridge-stdio-${args.runtimeScope}.pid` + : STDIO_PID_FILENAME; + const stdioPidPath = pidFilePath(args.agent, stdioPidFilename); const ownsViewerPort = args.daemon || !args.noViewer; const removeOwnedPidFile = () => { if (ownsViewerPort) removePidFile(pidPath); diff --git a/apps/memos-local-plugin/tests/python/test_bridge_client.py b/apps/memos-local-plugin/tests/python/test_bridge_client.py index 673961c3f..488632e85 100644 --- a/apps/memos-local-plugin/tests/python/test_bridge_client.py +++ b/apps/memos-local-plugin/tests/python/test_bridge_client.py @@ -44,6 +44,7 @@ class FakePopen: def __init__(self, *_args, **_kwargs) -> None: self.cmd = list(_args[0]) if _args else [] + self.env = dict(_kwargs.get("env") or {}) self.pid = 12345 self.stdin = io.StringIO() self._stdin_lines: list[str] = [] @@ -346,6 +347,23 @@ def test_module_singleton_independent_for_distinct_agents(self) -> None: hermes.close() openclaw.close() + def test_module_singleton_isolated_for_distinct_runtime_homes(self) -> None: + """Different MemOS data homes must not displace each other's client.""" + with tempfile.TemporaryDirectory() as root: + first = MemosBridgeClient( + bridge_path="/tmp/bridge.cts", + runtime_home=str(Path(root) / "home-a"), + ) + second = MemosBridgeClient( + bridge_path="/tmp/bridge.cts", + runtime_home=str(Path(root) / "home-b"), + ) + + self.assertFalse(first._closed) + self.assertFalse(second._closed) + first.close() + second.close() + def test_close_unregisters_active_client_only_when_still_current(self) -> None: """A stale close() must not evict the newer registered client.""" first = MemosBridgeClient(bridge_path="/tmp/bridge.cts") @@ -353,7 +371,11 @@ def test_close_unregisters_active_client_only_when_still_current(self) -> None: # First was already closed by second's __init__. Closing it again is # a no-op and must not touch the registry's current entry (second). first.close() - key = (second._singleton_agent, second._singleton_no_viewer) + key = ( + second._singleton_agent, + second._singleton_no_viewer, + second._singleton_runtime_home, + ) self.assertIs(bridge_client_mod._ACTIVE_CLIENTS.get(key), second) second.close() self.assertIsNone(bridge_client_mod._ACTIVE_CLIENTS.get(key)) @@ -365,6 +387,41 @@ def test_stdio_bridge_starts_without_viewer_by_default(self) -> None: self.assertIn("--no-viewer", cmd) client.close() + def test_stdio_bridge_passes_stable_runtime_scope(self) -> None: + with tempfile.TemporaryDirectory() as root: + client = MemosBridgeClient( + bridge_path="/tmp/bridge.cts", + runtime_home=root, + ) + assert self._fake is not None + scope_args = [ + arg for arg in getattr(self._fake, "cmd", []) + if arg.startswith("--runtime-scope=") + ] + self.assertEqual(len(scope_args), 1) + token = scope_args[0].split("=", 1)[1] + self.assertRegex(token, r"^[a-f0-9]{24}$") + self.assertEqual( + self._fake.env["MEMOS_HOME"], + str(Path(root).resolve()), + ) + client.close() + + def test_runtime_home_uses_captured_child_environment(self) -> None: + with tempfile.TemporaryDirectory() as root: + runtime_home = str(Path(root) / "captured-home") + client = MemosBridgeClient( + bridge_path="/tmp/bridge.cts", + extra_env={"MEMOS_HOME": runtime_home}, + ) + assert self._fake is not None + self.assertEqual( + client._singleton_runtime_home, + str(Path(runtime_home).resolve()), + ) + self.assertEqual(self._fake.env["MEMOS_HOME"], runtime_home) + client.close() + def test_reverse_request_waits_for_late_host_handler_registration(self) -> None: client = MemosBridgeClient(bridge_path="/tmp/bridge.cts") assert self._fake is not None diff --git a/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py b/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py index 445af36c6..02b78b307 100644 --- a/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py +++ b/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py @@ -204,7 +204,7 @@ def test_initialize_closes_previous_bridge_before_spawning_new_one(self) -> None patch("memos_provider.ensure_viewer_daemon", return_value=True), patch( "memos_provider.MemosBridgeClient", - side_effect=lambda: bridge_attempts.pop(0), + side_effect=lambda **_kwargs: bridge_attempts.pop(0), ), ): provider = memos_provider.MemTensorProvider() @@ -269,7 +269,7 @@ def close(self) -> None: patch("memos_provider.ensure_viewer_daemon", return_value=True), patch( "memos_provider.MemosBridgeClient", - side_effect=lambda: bridge_attempts.pop(0), + side_effect=lambda **_kwargs: bridge_attempts.pop(0), ), ): provider = memos_provider.MemTensorProvider() @@ -290,7 +290,7 @@ def test_sync_turn_recovers_when_initial_bridge_open_timed_out(self) -> None: recovered_bridge = FakeBridge() bridge_attempts = [failed_bridge, recovered_bridge] - def bridge_factory() -> FakeBridge: + def bridge_factory(**_kwargs: object) -> FakeBridge: return bridge_attempts.pop(0) with ( @@ -326,7 +326,10 @@ def test_delegation_recovers_when_initial_bridge_open_timed_out(self) -> None: with ( patch("memos_provider.ensure_bridge_running", return_value=True), patch("memos_provider.ensure_viewer_daemon", return_value=True), - patch("memos_provider.MemosBridgeClient", side_effect=lambda: bridge_attempts.pop(0)), + patch( + "memos_provider.MemosBridgeClient", + side_effect=lambda **_kwargs: bridge_attempts.pop(0), + ), ): provider = memos_provider.MemTensorProvider() provider.initialize("slow-parent-session") diff --git a/apps/memos-local-plugin/tests/python/test_shared_bridge_runtime.py b/apps/memos-local-plugin/tests/python/test_shared_bridge_runtime.py index b74be2f54..8247cbbd9 100644 --- a/apps/memos-local-plugin/tests/python/test_shared_bridge_runtime.py +++ b/apps/memos-local-plugin/tests/python/test_shared_bridge_runtime.py @@ -99,7 +99,7 @@ def tearDown(self) -> None: def test_two_leases_share_one_bridge_and_release_does_not_close_it(self) -> None: created: list[FakeBridge] = [] - def factory() -> FakeBridge: + def factory(*_args: object, **_kwargs: object) -> FakeBridge: bridge = FakeBridge() created.append(bridge) return bridge @@ -120,7 +120,7 @@ def factory() -> FakeBridge: def test_concurrent_reconnect_creates_one_replacement(self) -> None: created: list[FakeBridge] = [] - def factory() -> FakeBridge: + def factory(*_args: object, **_kwargs: object) -> FakeBridge: bridge = FakeBridge() created.append(bridge) return bridge @@ -294,7 +294,7 @@ def tearDown(self) -> None: def test_two_providers_share_physical_bridge_and_keep_sessions_isolated(self) -> None: created: list[FakeBridge] = [] - def factory() -> FakeBridge: + def factory(*_args: object, **_kwargs: object) -> FakeBridge: bridge = FakeBridge() created.append(bridge) return bridge @@ -303,7 +303,7 @@ def factory() -> FakeBridge: patch("memos_provider.MemosBridgeClient", side_effect=factory), patch("memos_provider.ensure_bridge_running", return_value=True), patch("memos_provider.ensure_viewer_daemon", return_value=True), - patch("memos_provider.kill_zombie_bridges", return_value=0), + patch("memos_provider.kill_zombie_bridges", return_value=0) as kill_zombies, ): first = memos_provider.MemTensorProvider() second = memos_provider.MemTensorProvider() @@ -317,6 +317,7 @@ def factory() -> FakeBridge: second.prefetch("remember B") first.sync_turn("remember A", "stored A") second.sync_turn("remember B", "stored B") + kill_zombies.assert_not_called() first.shutdown() self.assertFalse(created[0].closed) @@ -332,10 +333,188 @@ def factory() -> FakeBridge: self.assertIn("session-a", session_ids) self.assertIn("session-b", session_ids) + def test_lazy_provider_acquire_reuses_healthy_shared_bridge(self) -> None: + created: list[FakeBridge] = [] + + def factory(*_args: object, **_kwargs: object) -> FakeBridge: + bridge = FakeBridge() + created.append(bridge) + return bridge + + with ( + patch("memos_provider.MemosBridgeClient", side_effect=factory), + patch("memos_provider.ensure_bridge_running", return_value=True), + patch("memos_provider.ensure_viewer_daemon", return_value=True), + patch("memos_provider.kill_zombie_bridges", return_value=0), + ): + first = memos_provider.MemTensorProvider() + first.initialize("session-a") + assert first._bridge is not None + original_pid = first._bridge.pid + original_generation = first._bridge.generation + + second = memos_provider.MemTensorProvider() + second._session_id = "session-b" + self.assertTrue(second._ensure_bridge("session-b")) + + self.assertEqual(len(created), 1) + assert second._bridge is not None + self.assertEqual(second._bridge.pid, original_pid) + self.assertEqual(second._bridge.generation, original_generation) + self.assertEqual(second._bridge.status()["leases"], 2) + + first.shutdown() + second.shutdown() + + def test_distinct_data_home_runtime_keeps_its_spawn_environment(self) -> None: + created: list[FakeBridge] = [] + client_kwargs: list[dict[str, object]] = [] + + def factory(*_args: object, **kwargs: object) -> FakeBridge: + client_kwargs.append(dict(kwargs)) + bridge = FakeBridge() + created.append(bridge) + return bridge + + with ( + patch("memos_provider.MemosBridgeClient", side_effect=factory), + patch("memos_provider.ensure_bridge_running", return_value=True), + patch("memos_provider.ensure_viewer_daemon", return_value=True), + patch("memos_provider.kill_zombie_bridges", return_value=0), + ): + first = memos_provider.MemTensorProvider() + second = memos_provider.MemTensorProvider() + with patch.dict("os.environ", {"MEMOS_HOME": "/tmp/memos-home-a"}): + first.initialize("session-a") + with patch.dict("os.environ", {"MEMOS_HOME": "/tmp/memos-home-b"}): + second.initialize("session-b") + + self.assertEqual(len(created), 2) + home_a = str(Path("/tmp/memos-home-a").resolve()) + home_b = str(Path("/tmp/memos-home-b").resolve()) + self.assertEqual(client_kwargs[0]["runtime_home"], home_a) + self.assertEqual(client_kwargs[1]["runtime_home"], home_b) + + assert first._bridge is not None + with patch.dict("os.environ", {"MEMOS_HOME": "/tmp/memos-home-b"}): + first._bridge.reconnect(expected_generation=first._bridge.generation) + + self.assertEqual(len(created), 3) + self.assertEqual(client_kwargs[2]["runtime_home"], home_a) + self.assertEqual( + client_kwargs[2]["extra_env"], + {"MEMOS_HOME": home_a}, + ) + + first.shutdown() + second.shutdown() + + def test_lazy_acquire_failure_releases_new_lease(self) -> None: + class RejectSessionBridge(FakeBridge): + def request( + self, + method: str, + params: dict | None = None, + **kwargs: object, + ) -> dict: + if method == "session.open": + raise BridgeError("internal", "session rejected") + return super().request(method, params, **kwargs) + + with ( + patch( + "memos_provider.MemosBridgeClient", + side_effect=lambda *_args, **_kwargs: RejectSessionBridge(), + ), + patch("memos_provider.ensure_bridge_running", return_value=True), + patch("memos_provider.ensure_viewer_daemon", return_value=True), + patch("memos_provider.kill_zombie_bridges", return_value=0), + ): + provider = memos_provider.MemTensorProvider() + provider._session_id = "session-a" + + self.assertFalse(provider._ensure_bridge("session-a")) + self.assertIsNone(provider._bridge) + statuses = memos_provider.SHARED_BRIDGE_REGISTRY.status() + self.assertEqual(len(statuses), 1) + self.assertEqual(statuses[0]["leases"], 0) + + provider.shutdown() + + def test_lazy_acquire_reconnects_once_on_transport_closed(self) -> None: + created: list[FakeBridge] = [] + + class FirstSessionOpenFails(FakeBridge): + def request( + self, + method: str, + params: dict | None = None, + **kwargs: object, + ) -> dict: + if method == "session.open": + raise BridgeError("transport_closed", "bridge subprocess exited") + return super().request(method, params, **kwargs) + + def factory(*_args: object, **_kwargs: object) -> FakeBridge: + bridge: FakeBridge + if not created: + bridge = FirstSessionOpenFails() + else: + bridge = FakeBridge() + created.append(bridge) + return bridge + + with ( + patch("memos_provider.MemosBridgeClient", side_effect=factory), + patch("memos_provider.ensure_bridge_running", return_value=True), + patch("memos_provider.ensure_viewer_daemon", return_value=True), + patch("memos_provider.kill_zombie_bridges", return_value=0), + ): + provider = memos_provider.MemTensorProvider() + provider._session_id = "session-a" + + self.assertTrue(provider._ensure_bridge("session-a")) + self.assertEqual(len(created), 2) + self.assertTrue(created[0].closed) + assert provider._bridge is not None + self.assertEqual(provider._bridge.generation, 2) + self.assertEqual(provider._bridge.status()["leases"], 1) + + provider.shutdown() + + def test_existing_lease_is_retained_when_reconnect_fails(self) -> None: + attempts = 0 + + def factory(*_args: object, **_kwargs: object) -> FakeBridge: + nonlocal attempts + attempts += 1 + if attempts > 1: + raise RuntimeError("replacement spawn failed") + return FakeBridge() + + with ( + patch("memos_provider.MemosBridgeClient", side_effect=factory), + patch("memos_provider.ensure_bridge_running", return_value=True), + patch("memos_provider.ensure_viewer_daemon", return_value=True), + patch("memos_provider.kill_zombie_bridges", return_value=0), + ): + provider = memos_provider.MemTensorProvider() + provider.initialize("session-a") + original_lease = provider._bridge + assert original_lease is not None + + with self.assertRaises(RuntimeError): + provider._reconnect_bridge("session-a") + + self.assertIs(provider._bridge, original_lease) + self.assertEqual(original_lease.status()["leases"], 1) + + provider.shutdown() + def test_provider_reopens_its_session_after_another_lease_reconnects(self) -> None: created: list[FakeBridge] = [] - def factory() -> FakeBridge: + def factory(*_args: object, **_kwargs: object) -> FakeBridge: bridge = FakeBridge() created.append(bridge) return bridge From 64b1cd9569dd22bb20a58c8f213fbb124fd274ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=81=E5=9C=A8=E5=90=B5=E7=9D=80=E5=90=83=E7=B3=96?= Date: Wed, 29 Jul 2026 11:13:58 +0800 Subject: [PATCH 06/18] fix(plugin): harden Hermes recovery lifecycle --- apps/memos-local-plugin/bridge.cts | 12 +- apps/memos-local-plugin/bridge.mts | 12 +- .../core/capture/capture.ts | 34 +++ apps/memos-local-plugin/core/logger/README.md | 5 +- apps/memos-local-plugin/core/logger/index.ts | 9 +- .../core/pipeline/memory-core.ts | 276 ++++++++++++------ apps/memos-local-plugin/core/reward/reward.ts | 5 + .../memos-local-plugin/server/routes/admin.ts | 54 +++- apps/memos-local-plugin/server/types.ts | 14 + .../tests/unit/capture/capture.test.ts | 85 +++++- .../unit/logger/signal-ownership.test.ts | 23 ++ .../unit/pipeline/auto-recovery-role.test.ts | 147 ++++++++++ .../tests/unit/pipeline/memory-core.test.ts | 176 ++++++++++- .../unit/reward/reward.integration.test.ts | 64 ++++ .../tests/unit/server/admin.test.ts | 54 +++- 15 files changed, 848 insertions(+), 122 deletions(-) create mode 100644 apps/memos-local-plugin/tests/unit/logger/signal-ownership.test.ts create mode 100644 apps/memos-local-plugin/tests/unit/pipeline/auto-recovery-role.test.ts diff --git a/apps/memos-local-plugin/bridge.cts b/apps/memos-local-plugin/bridge.cts index 86bcdd07a..acc1ccb29 100644 --- a/apps/memos-local-plugin/bridge.cts +++ b/apps/memos-local-plugin/bridge.cts @@ -302,6 +302,7 @@ async function main(): Promise { const { core, config, home } = await bootstrapMemoryCoreFull({ agent: args.agent, namespace: { agentKind: args.agent, profileId: resolvedProfileId }, + autoRecovery: args.agent !== "hermes" || !args.daemon, pkgVersion, hostLlmBridge: args.daemon ? null : lazyHostLlmBridge, home: resolvedHome, @@ -494,8 +495,15 @@ async function main(): Promise { process.stderr.write(`bridge: daemon received ${sig}, shutting down\n`); removeOwnedPidFile(); try { await viewer!.close(); } catch { /* best-effort */ } - await withShutdownTimeout(core.shutdown()); - process.exit(0); + try { + await withShutdownTimeout(core.shutdown()); + } catch { + // clear-data already shuts the core down before removing SQLite. + // The signal still has to terminate the daemon so the supervisor + // can replace it. + } finally { + process.exit(0); + } }; process.on("SIGINT", () => void shutdownDaemon("SIGINT")); process.on("SIGTERM", () => void shutdownDaemon("SIGTERM")); diff --git a/apps/memos-local-plugin/bridge.mts b/apps/memos-local-plugin/bridge.mts index 95b15e21c..2d967c597 100644 --- a/apps/memos-local-plugin/bridge.mts +++ b/apps/memos-local-plugin/bridge.mts @@ -245,6 +245,7 @@ async function main(): Promise { const { core, config, home } = await bootstrapMemoryCoreFull({ agent: args.agent, namespace: { agentKind: args.agent, profileId: "default" }, + autoRecovery: args.agent !== "hermes" || !args.daemon, pkgVersion, hostLlmBridge: args.daemon ? null : lazyHostLlmBridge, home: resolvedHome, @@ -407,8 +408,15 @@ async function main(): Promise { process.stderr.write(`bridge: daemon received ${sig}, shutting down\n`); removeOwnedPidFile(); try { await viewer!.close(); } catch { /* best-effort */ } - await core.shutdown(); - process.exit(0); + try { + await core.shutdown(); + } catch { + // clear-data already shuts the core down before removing SQLite. + // The signal still has to terminate the daemon so the supervisor + // can replace it. + } finally { + process.exit(0); + } }; process.on("SIGINT", () => void shutdownDaemon("SIGINT")); process.on("SIGTERM", () => void shutdownDaemon("SIGTERM")); diff --git a/apps/memos-local-plugin/core/capture/capture.ts b/apps/memos-local-plugin/core/capture/capture.ts index 576ba9060..94be58fff 100644 --- a/apps/memos-local-plugin/core/capture/capture.ts +++ b/apps/memos-local-plugin/core/capture/capture.ts @@ -410,6 +410,40 @@ export function createCaptureRunner(deps: CaptureDeps): CaptureRunner { const recoveredReplay = isRecoveredReplay(input.episode); const matcher = createTraceMatcher(existing, { allowRelaxedToolTiming: recoveredReplay }); const matchedRows = normalized.map((s) => matcher.take(s)); + // A recovered snapshot is reconstructed from the trace rows that we are + // about to patch. If it produces steps but none of those steps match any + // source row, continuing into orphan handling / reflection would replay + // an invalid snapshot, spend LLM calls, and finally emit capture.done + // with no usable trace ids. Fail before any of those side effects. + // + // Partial matches remain recoverable: their matching rows are patched + // and the unmatched tail follows the existing bounded-orphan policy. + if ( + recoveredReplay && + normalized.length > 0 && + matchedRows.every((row) => row === null) + ) { + const error = new MemosError( + ERROR_CODES.CONFLICT, + "recovered replay did not match any persisted trace rows", + { + episodeId: input.episode.id, + normalizedSteps: normalized.length, + persistedTraces: existing.length, + }, + ); + emit({ + kind: "capture.failed", + episodeId: input.episode.id, + sessionId: input.episode.sessionId, + stage: "match", + error: { + code: ERROR_CODES.CONFLICT, + message: error.message, + }, + }); + throw error; + } const orphanEntries = normalized .map((step, index) => ({ step, index })) .filter(({ index }) => matchedRows[index] === null); diff --git a/apps/memos-local-plugin/core/logger/README.md b/apps/memos-local-plugin/core/logger/README.md index 5504a0915..9ccd9394e 100644 --- a/apps/memos-local-plugin/core/logger/README.md +++ b/apps/memos-local-plugin/core/logger/README.md @@ -86,8 +86,9 @@ per-channel notes. Example prefixes: - **Disk full / permission denied** — file transports degrade to console only and emit one error to the in-memory ring buffer (also surfaced via SSE). -- **Process abort** — every transport `flushSync()`s on `beforeExit` and - `SIGINT`/`SIGTERM`. +- **Process exit** — transports close on `beforeExit`. Executable bridge + entries own `SIGINT`/`SIGTERM` so they can drain MemoryCore before exit; + file writes themselves are synchronous. - **Audit retention** — never deletes. If you really need to free space, gzip archives can be moved out of `logs/` manually. - **Redaction is best-effort** — exotic key names won't be caught. If you have diff --git a/apps/memos-local-plugin/core/logger/index.ts b/apps/memos-local-plugin/core/logger/index.ts index d9ac5afb0..897ffe4b1 100644 --- a/apps/memos-local-plugin/core/logger/index.ts +++ b/apps/memos-local-plugin/core/logger/index.ts @@ -411,7 +411,7 @@ function round2(n: number): number { return Math.round(n * 100) / 100; } -// ─── Process-exit hooks (only register once) ────────────────────────────── +// ─── Process-exit hook (only register once) ─────────────────────────────── let exitHooked = false; function hookProcessExit(): void { @@ -419,8 +419,11 @@ function hookProcessExit(): void { exitHooked = true; const onExit = () => { void shutdownLogger(); }; process.once("beforeExit", onExit); - process.once("SIGINT", () => { onExit(); process.exit(130); }); - process.once("SIGTERM", () => { onExit(); process.exit(143); }); + // Deliberately do not own SIGINT/SIGTERM here. The dedicated bridge + // entries drain their HTTP/stdin handles and MemoryCore before exiting; + // a library-level process.exit() used to pre-empt those handlers and + // leave episodes/SQLite work half-finished. Embedded hosts likewise own + // their process lifecycle. } // ─── Public root logger ──────────────────────────────────────────────────── diff --git a/apps/memos-local-plugin/core/pipeline/memory-core.ts b/apps/memos-local-plugin/core/pipeline/memory-core.ts index bc0c7e689..ddcdf1602 100644 --- a/apps/memos-local-plugin/core/pipeline/memory-core.ts +++ b/apps/memos-local-plugin/core/pipeline/memory-core.ts @@ -138,6 +138,12 @@ type DedicatedLlmConfig = { export interface BootstrapOptions { agent: AgentKind; namespace?: RuntimeNamespace; + /** + * Run automatic startup/periodic episode recovery. Defaults to true. + * Hermes disables this for its read-oriented Viewer daemon so only the + * stdio bridge owns recovery writes for a given data home. + */ + autoRecovery?: boolean; /** Optional pre-resolved home. If omitted, derived from `resolveHome`. */ home?: ResolvedHome; /** Optional pre-resolved config. If omitted, we load from disk. */ @@ -526,6 +532,7 @@ export async function bootstrapMemoryCoreFull( const handle = createPipeline(deps); const core = createMemoryCore(handle, home, options.pkgVersion ?? "dev", { + autoRecovery: options.autoRecovery ?? true, telemetry: options.telemetry ?? null, onShutdown: () => { try { @@ -544,6 +551,11 @@ export async function bootstrapMemoryCoreFull( // ─── Facade factory ────────────────────────────────────────────────────────── export interface CreateMemoryCoreOptions { + /** + * Run automatic startup/periodic episode recovery. Defaults to true. + * This does not disable normal turn capture or explicit repair operations. + */ + autoRecovery?: boolean; /** Called after the pipeline has shut down. */ onShutdown?: () => void | Promise; /** Optional telemetry instance for ARMS RUM reporting. */ @@ -566,6 +578,7 @@ export function createMemoryCore( ): MemoryCore { const bootAt = Date.now(); const log = rootLogger.child({ channel: "core.pipeline.memory-core" }); + const autoRecoveryEnabled = options.autoRecovery ?? true; let telemetry = options.telemetry ?? null; let initialized = false; let shutDown = false; @@ -675,6 +688,7 @@ export function createMemoryCore( let lastStaleScan = 0; let lastDirtyClosedScan = 0; async function autoFinalizeStaleTasks(): Promise { + if (!autoRecoveryEnabled) return; const nowMs = Date.now(); if (nowMs - lastStaleScan < 30_000) return; lastStaleScan = nowMs; @@ -735,6 +749,7 @@ export function createMemoryCore( } async function autoRescoreDirtyClosedEpisodes(): Promise { + if (!autoRecoveryEnabled) return; const nowMs = Date.now(); if (nowMs - lastDirtyClosedScan < 30_000) return; lastDirtyClosedScan = nowMs; @@ -1037,7 +1052,8 @@ export function createMemoryCore( // call `core.waitForStartupRecovery()` after `init()`. let staleForBackground: Array }> = []; let dirtyClosedForBackground: Array }> = []; - try { + if (autoRecoveryEnabled) { + try { const orphans = handle.repos.episodes.list({ status: "open", limit: 500 }); if (orphans.length > 0) { const nowMs = Date.now(); @@ -1141,16 +1157,20 @@ export function createMemoryCore( } dirtyClosedForBackground = dirtyClosed; } - } catch (err) { - log.debug("init.orphan_scan.failed", { - err: err instanceof Error ? err.message : String(err), - }); + } catch (err) { + log.debug("init.orphan_scan.failed", { + err: err instanceof Error ? err.message : String(err), + }); + } } // Kick the slow recovery chain off the main thread. `init()` returns // as soon as the synchronous classification above finishes, so the // Gateway can start accepting WebSocket upgrades immediately. - if (staleForBackground.length > 0 || dirtyClosedForBackground.length > 0) { + if ( + autoRecoveryEnabled && + (staleForBackground.length > 0 || dirtyClosedForBackground.length > 0) + ) { const stale = staleForBackground; const dirtyClosed = dirtyClosedForBackground; log.info("init.background_recovery_started", { @@ -1184,15 +1204,17 @@ export function createMemoryCore( // Periodic rescore timer for episodes that miss the startup scan or // retry of failed reward runs. 10-minute interval is safe because // autoRescoreDirtyClosedEpisodes has its own 30-second dedup guard. - const rescoreInterval = setInterval(() => { - void autoRescoreDirtyClosedEpisodes().catch((err) => { - log.debug("periodic_rescore.error", { - err: err instanceof Error ? err.message : String(err), + if (autoRecoveryEnabled) { + const rescoreInterval = setInterval(() => { + void autoRescoreDirtyClosedEpisodes().catch((err) => { + log.debug("periodic_rescore.error", { + err: err instanceof Error ? err.message : String(err), + }); }); - }); - }, 10 * 60 * 1000); - // Mark as unref so the timer doesn't block shutdown - (rescoreInterval as unknown as { unref?: () => void }).unref?.(); + }, 10 * 60 * 1000); + // Mark as unref so the timer doesn't block shutdown + (rescoreInterval as unknown as { unref?: () => void }).unref?.(); + } // Wire `memory_add` into the api_logs table on EVERY turn so the // Logs viewer shows per-turn capture activity. `capture.lite.done` @@ -1416,92 +1438,119 @@ export function createMemoryCore( }); const needsRewardFallback: EpisodeId[] = []; - for (const ep of orphans) { - if (isLightweightEpisode(ep)) continue; - try { - const episodeId = ep.id as EpisodeId; - const traceIds = (ep.traceIds ?? []) as TraceId[]; - handle.repos.episodes.close(episodeId, endedAt, ep.rTask ?? undefined); - handle.repos.episodes.updateMeta(episodeId, { - closeReason: "finalized", - abandonReason: undefined, - recoveredAtStartup: endedAt, - recoveryReason: "missed_session_end", - }); + // Scope capture failures to this startup-recovery batch. Normal live + // captures and explicit/manual reward runs must retain their existing + // behaviour. + const captureFailedInBatch = new Set(); + const recoveryEpisodeIds = new Set( + orphans.map((ep) => ep.id as EpisodeId), + ); + const unsubscribeCaptureFailed = handle.buses.capture.on("capture.failed", (evt) => { + if ( + evt.kind === "capture.failed" && + recoveryEpisodeIds.has(evt.episodeId) + ) { + captureFailedInBatch.add(evt.episodeId); + } + }); + try { + for (const ep of orphans) { + if (isLightweightEpisode(ep)) continue; + try { + const episodeId = ep.id as EpisodeId; + const traceIds = (ep.traceIds ?? []) as TraceId[]; + handle.repos.episodes.close(episodeId, endedAt, ep.rTask ?? undefined); + handle.repos.episodes.updateMeta(episodeId, { + closeReason: "finalized", + abandonReason: undefined, + recoveredAtStartup: endedAt, + recoveryReason: "missed_session_end", + }); - if (ep.rTask != null && !episodeRewardIsDirty(ep)) { - log.info("init.orphan.repaired_finalized", { + if (ep.rTask != null && !episodeRewardIsDirty(ep)) { + log.info("init.orphan.repaired_finalized", { + episodeId, + sessionId: ep.sessionId, + rTask: ep.rTask, + }); + debugStartupRecovery("H2", "startup_recovery_already_scored", { + episodeId, + sessionId: ep.sessionId, + rTask: ep.rTask, + }); + continue; + } + + const snapshot = snapshotFromRecoveredEpisode(ep, endedAt); + debugStartupRecovery("H3", "startup_recovery_emit_finalized", { episodeId, sessionId: ep.sessionId, - rTask: ep.rTask, + traceCount: traceIds.length, + recoveredTurnCount: snapshot.turnCount, }); - debugStartupRecovery("H2", "startup_recovery_already_scored", { - episodeId, - sessionId: ep.sessionId, - rTask: ep.rTask, + handle.buses.session.emit({ + kind: "episode.finalized", + episode: snapshot, + closedBy: "finalized", + }); + needsRewardFallback.push(episodeId); + } catch (err) { + log.debug("init.orphan_recovery.skipped", { + episodeId: ep.id, + err: err instanceof Error ? err.message : String(err), + }); + debugStartupRecovery("H4", "startup_recovery_error", { + episodeId: ep.id, + err: err instanceof Error ? err.message : String(err), }); - continue; } + } - const snapshot = snapshotFromRecoveredEpisode(ep, endedAt); - debugStartupRecovery("H3", "startup_recovery_emit_finalized", { - episodeId, - sessionId: ep.sessionId, - traceCount: traceIds.length, - recoveredTurnCount: snapshot.turnCount, - }); - handle.buses.session.emit({ - kind: "episode.finalized", - episode: snapshot, - closedBy: "finalized", + try { + await handle.flush(); + for (const episodeId of needsRewardFallback) { + if (captureFailedInBatch.has(episodeId)) { + log.warn("init.orphan_recovery.reward_fallback_skipped", { + episodeId, + reason: "capture_failed", + }); + continue; + } + const row = handle.repos.episodes.getById(episodeId); + if (row?.rTask == null) { + await handle.rewardRunner.run({ + episodeId, + feedback: [], + trigger: "manual", + }); + } + } + await handle.flush(); + debugStartupRecovery("H5", "startup_recovery_flush_done", { + recoveredCount: orphans.length, + rewardedEpisodes: needsRewardFallback.map((episodeId) => { + const row = handle.repos.episodes.getById(episodeId); + return { + episodeId, + rTask: row?.rTask ?? null, + captureFailed: captureFailedInBatch.has(episodeId), + closeReason: (row?.meta as { closeReason?: unknown } | undefined)?.closeReason, + abandonReason: (row?.meta as { abandonReason?: unknown } | undefined)?.abandonReason, + }; + }), }); - needsRewardFallback.push(episodeId); } catch (err) { - log.debug("init.orphan_recovery.skipped", { - episodeId: ep.id, + log.warn("init.orphan_recovery.flush_failed", { + count: orphans.length, err: err instanceof Error ? err.message : String(err), }); - debugStartupRecovery("H4", "startup_recovery_error", { - episodeId: ep.id, + debugStartupRecovery("H5", "startup_recovery_flush_failed", { + count: orphans.length, err: err instanceof Error ? err.message : String(err), }); } - } - - try { - await handle.flush(); - for (const episodeId of needsRewardFallback) { - const row = handle.repos.episodes.getById(episodeId); - if (row?.rTask == null) { - await handle.rewardRunner.run({ - episodeId, - feedback: [], - trigger: "manual", - }); - } - } - await handle.flush(); - debugStartupRecovery("H5", "startup_recovery_flush_done", { - recoveredCount: orphans.length, - rewardedEpisodes: needsRewardFallback.map((episodeId) => { - const row = handle.repos.episodes.getById(episodeId); - return { - episodeId, - rTask: row?.rTask ?? null, - closeReason: (row?.meta as { closeReason?: unknown } | undefined)?.closeReason, - abandonReason: (row?.meta as { abandonReason?: unknown } | undefined)?.abandonReason, - }; - }), - }); - } catch (err) { - log.warn("init.orphan_recovery.flush_failed", { - count: orphans.length, - err: err instanceof Error ? err.message : String(err), - }); - debugStartupRecovery("H5", "startup_recovery_flush_failed", { - count: orphans.length, - err: err instanceof Error ? err.message : String(err), - }); + } finally { + unsubscribeCaptureFailed(); } } @@ -1590,6 +1639,27 @@ export function createMemoryCore( id: last.id, }); } + // Older builds could leave both `reward.skipped=true` and a stale + // `rewardDirty` marker on short/trivial episodes. The skip is terminal; + // clean the contradictory marker while this bounded page is already in + // memory so upgraded installations stop retrying those rows immediately. + for (const ep of page) { + if (!rewardWasSkipped(ep) || !hasRewardDirtyMarker(ep)) continue; + try { + handle.repos.episodes.updateMeta(ep.id as EpisodeId, { + rewardDirty: undefined, + }); + ep.meta = { ...(ep.meta ?? {}), rewardDirty: undefined }; + } catch (err) { + // A cleanup write must not prevent the rest of this page from being + // classified and recovered. The terminal skip still keeps this row + // out of the retry queue, and a later bounded scan can clean it again. + log.debug("dirty_closed_reward.skip_marker_cleanup_error", { + episodeId: ep.id, + err: err instanceof Error ? err.message : String(err), + }); + } + } return page; } @@ -1605,12 +1675,12 @@ export function createMemoryCore( function episodeRewardIsDirty(ep: EpisodeRow & { meta?: Record }): boolean { const meta = ep.meta ?? {}; if (meta.lightweightMemory === true) return false; - if (meta.rewardDirty && typeof meta.rewardDirty === "object") return true; + // An intentional triviality skip is terminal and must take precedence + // over a stale retry marker written by an older recovery attempt. + if (rewardWasSkipped(ep)) return false; + if (hasRewardDirtyMarker(ep)) return true; const reward = meta.reward; - if (reward && typeof reward === "object" && (reward as { skipped?: unknown }).skipped === true) { - return false; - } if ( ep.rTask == null && (ep.traceIds?.length ?? 0) > 0 && @@ -1702,10 +1772,14 @@ export function createMemoryCore( : []; const turns: EpisodeTurn[] = []; for (const tr of traces) { + const turnId = + typeof tr.turnId === "number" && Number.isFinite(tr.turnId) + ? (tr.turnId as EpochMs) + : tr.ts; if (tr.userText) { turns.push({ id: `${tr.id}:user`, - ts: tr.ts, + ts: turnId, role: "user", content: tr.userText, }); @@ -4536,6 +4610,26 @@ export function createMemoryCore( return true; } + function rewardWasSkipped( + ep: EpisodeRow & { meta?: Record }, + ): boolean { + const reward = ep.meta?.reward; + return Boolean( + reward && + typeof reward === "object" && + (reward as { skipped?: unknown }).skipped === true, + ); + } + + function hasRewardDirtyMarker( + ep: EpisodeRow & { meta?: Record }, + ): boolean { + return Boolean( + ep.meta?.rewardDirty && + typeof ep.meta.rewardDirty === "object", + ); + } + function collectEmbeddingSlots(): EmbeddingSlot[] { const slots: EmbeddingSlot[] = []; const pageSize = 500; diff --git a/apps/memos-local-plugin/core/reward/reward.ts b/apps/memos-local-plugin/core/reward/reward.ts index 1b6d3354c..0da87e5cb 100644 --- a/apps/memos-local-plugin/core/reward/reward.ts +++ b/apps/memos-local-plugin/core/reward/reward.ts @@ -128,6 +128,11 @@ export function createRewardRunner(deps: RewardDeps): RewardRunner { trigger: input.trigger, skipped: true, }, + // A trivial/short episode is a terminal, intentional skip rather + // than a failed reward run. Clear any recovery marker inherited + // from an earlier capture/reward attempt so automatic rescans do + // not keep reflecting the same trace forever. + rewardDirty: undefined, }); } catch (err) { warnings.push({ diff --git a/apps/memos-local-plugin/server/routes/admin.ts b/apps/memos-local-plugin/server/routes/admin.ts index 29d38bcde..f70d52b77 100644 --- a/apps/memos-local-plugin/server/routes/admin.ts +++ b/apps/memos-local-plugin/server/routes/admin.ts @@ -12,8 +12,9 @@ * Agent-aware restart. For OpenClaw the plugin lives inside the * gateway process, which is managed by macOS launchd — calling * `process.exit(0)` causes launchd to respawn it automatically. - * For Hermes, terminate the active `hermes chat`, spawn a replacement - * viewer daemon, then exit so both processes reload the saved config. + * For Hermes, terminate the active `hermes chat`, then ask the bridge + * to shut down gracefully. launchd/systemd owns replacement when the + * viewer is supervised; portable viewers retain the detached fallback. */ import { spawn } from "node:child_process"; import type { ServerDeps, ServerOptions } from "../types.js"; @@ -43,11 +44,15 @@ export function registerAdminRoutes(routes: Routes, deps: ServerDeps, options: S if (agent === "hermes" && deps.home?.root) { try { await fs.unlink(`${deps.home.root}/bridge-status.json`); } catch { /* may not exist */ } } - if (agent !== "openclaw") { - // Hermes: spawn replacement daemon after clearing data + if (agent !== "openclaw" && !isSupervisorManaged(options)) { + // Portable Hermes: there is no supervisor to replace this process. await spawnReplacementDaemon(agent); } - setTimeout(() => process.exit(0), 200); + if (agent === "hermes") { + scheduleHermesShutdown(options, 200); + } else { + setTimeout(() => process.exit(0), 200); + } return { ok: true, restarting: true, killedHermes }; }); @@ -60,8 +65,10 @@ export function registerAdminRoutes(routes: Routes, deps: ServerDeps, options: S if (agent === "hermes") { const killed = await terminateHermesChat(); - await spawnReplacementDaemon(agent); - setTimeout(() => process.exit(0), 200); + if (!isSupervisorManaged(options)) { + await spawnReplacementDaemon(agent); + } + scheduleHermesShutdown(options, 200); return { ok: true, restarting: true, killed }; } @@ -69,6 +76,39 @@ export function registerAdminRoutes(routes: Routes, deps: ServerDeps, options: S }); } +/** + * Detect the supervisors used by supported desktop/server installs. + * + * The MemOS launchd job exports its stable label through XPC_SERVICE_NAME. + * Do not treat arbitrary GUI-app XPC labels as supervision: a portable + * daemon launched from such an app would exit with nobody to replace it. + * systemd services expose INVOCATION_ID for the invocation. + */ +export function isSupervisorManagedProcess( + env: NodeJS.ProcessEnv = process.env, +): boolean { + const xpcService = env.XPC_SERVICE_NAME?.trim(); + if (xpcService?.startsWith("ai.memtensor.memos-local-hermes")) return true; + return Boolean(env.INVOCATION_ID?.trim()); +} + +function isSupervisorManaged(options: ServerOptions): boolean { + return options.lifecycle?.supervised ?? isSupervisorManagedProcess(); +} + +function scheduleHermesShutdown(options: ServerOptions, delayMs: number): void { + setTimeout(() => { + if (options.lifecycle?.requestShutdown) { + options.lifecycle.requestShutdown(); + return; + } + // The bridge entry owns SIGINT/SIGTERM and drains the HTTP server, + // MemoryCore, telemetry and SQLite before exiting. Do not call + // process.exit() here: that bypasses the bridge's graceful path. + process.kill(process.pid, "SIGTERM"); + }, delayMs); +} + async function spawnReplacementDaemon(agent: string): Promise { const fs = await import("node:fs"); const nodePath = await import("node:path"); diff --git a/apps/memos-local-plugin/server/types.ts b/apps/memos-local-plugin/server/types.ts index 8532101ef..9667e8a99 100644 --- a/apps/memos-local-plugin/server/types.ts +++ b/apps/memos-local-plugin/server/types.ts @@ -39,6 +39,20 @@ export interface ServerOptions { * installed on disk. */ agent?: "openclaw" | "hermes"; + /** + * Process lifecycle hooks used by the admin restart route. + * + * A supervised Hermes viewer must let launchd/systemd perform the + * replacement; spawning a second daemon from the route races the + * supervisor's KeepAlive restart. Tests and portable embedders may + * override the auto-detection and shutdown action explicitly. + */ + lifecycle?: { + /** Override launchd/systemd detection. */ + supervised?: boolean; + /** Request graceful host shutdown after the HTTP response is returned. */ + requestShutdown?: () => void; + }; } export interface ServerHandle { diff --git a/apps/memos-local-plugin/tests/unit/capture/capture.test.ts b/apps/memos-local-plugin/tests/unit/capture/capture.test.ts index b30cbe4c1..e88a34f94 100644 --- a/apps/memos-local-plugin/tests/unit/capture/capture.test.ts +++ b/apps/memos-local-plugin/tests/unit/capture/capture.test.ts @@ -224,7 +224,7 @@ describe("capture/pipeline (end-to-end)", () => { it("recovered replay matches tool traces by payload when timestamps drift", async () => { tmp.repos.traces.insert(traceRow({ - id: "tr_tool", + id: "tr_tool_search", ts: 1_001, turnId: 1_000, userText: "run search", @@ -234,9 +234,19 @@ describe("capture/pipeline (end-to-end)", () => { output: "ok", }], })); + tmp.repos.traces.insert(traceRow({ + id: "tr_tool_read", + ts: 1_002, + turnId: 1_000, + toolCalls: [{ + name: "read", + input: { path: "/tmp/result" }, + output: "contents", + }], + })); tmp.repos.traces.insert(traceRow({ id: "tr_response", - ts: 1_003, + ts: 1_004, turnId: 1_000, agentText: "done", })); @@ -256,19 +266,70 @@ describe("capture/pipeline (end-to-end)", () => { input: { q: "memos" }, output: "ok", }), - turn("assistant", "done", 1_003), + turn("tool", "contents", 1_003, { + name: "read", + input: { path: "/tmp/result" }, + output: "contents", + }), + turn("assistant", "done", 1_004), ], }); - ep.meta = { recoveredAtStartup: 1_004, recoveryReason: "dirty_reward_rescore" }; + ep.meta = { recoveredAtStartup: 1_005, recoveryReason: "dirty_reward_rescore" }; const result = await runner.runReflect({ episode: ep, closedBy: "finalized" }); - expect(result.traceIds).toHaveLength(2); + expect(result.traceIds).toHaveLength(3); expect(result.warnings.some((w) => w.message.includes("skipped recovered orphan"))).toBe(false); - expect(tmp.repos.traces.listAllForEpisode("ep_1" as EpisodeId)).toHaveLength(2); + expect(tmp.repos.traces.listAllForEpisode("ep_1" as EpisodeId)).toHaveLength(3); }); - it("does not insert replay orphans for startup-recovered episodes by default", async () => { + it("fails a fully-unmatched recovered replay before LLM, inserts, or capture.done", async () => { + const llm = fakeLlm({ + complete: { + "capture.reflection.synth": "should never be requested", + }, + }); + const runner = buildRunner({ + alphaScoring: false, + synthReflections: true, + embedTraces: false, + }, llm, null); + const ep = episodeSnapshot({ + id: "ep_1", + sessionId: "se_1", + turns: [ + turn("user", "hello", 1_000), + turn("assistant", "hi", 1_010), + ], + }); + ep.meta = { recoveredAtStartup: 1_020, recoveryReason: "dirty_reward_rescore" }; + + await expect( + runner.runReflect({ episode: ep, closedBy: "finalized" }), + ).rejects.toMatchObject({ + name: "MemosError", + code: "conflict", + }); + + expect(tmp.repos.traces.listAllForEpisode("ep_1" as EpisodeId)).toHaveLength(0); + expect(llm.stats().requests).toBe(0); + expect(seen).toContainEqual(expect.objectContaining({ + kind: "capture.failed", + episodeId: "ep_1", + stage: "match", + error: expect.objectContaining({ code: "conflict" }), + })); + expect(seen.some((evt) => evt.kind === "capture.done")).toBe(false); + }); + + it("does not fail recovered replay when at least one trace matches", async () => { + tmp.repos.traces.insert(traceRow({ + id: "tr_matching", + ts: 1_010, + turnId: 1_000, + userText: "hello", + agentText: "hi", + })); const runner = buildRunner({ alphaScoring: false, synthReflections: false, @@ -280,15 +341,19 @@ describe("capture/pipeline (end-to-end)", () => { turns: [ turn("user", "hello", 1_000), turn("assistant", "hi", 1_010), + turn("user", "unmatched follow-up", 2_000), + turn("assistant", "unmatched answer", 2_010), ], }); - ep.meta = { recoveredAtStartup: 1_020, recoveryReason: "dirty_reward_rescore" }; + ep.meta = { recoveredAtStartup: 2_020, recoveryReason: "dirty_reward_rescore" }; const result = await runner.runReflect({ episode: ep, closedBy: "finalized" }); - expect(tmp.repos.traces.listAllForEpisode("ep_1" as EpisodeId)).toHaveLength(0); - expect(result.traceIds).toHaveLength(0); + expect(result.traceIds).toEqual(["tr_matching"]); expect(result.warnings.some((w) => w.message.includes("skipped recovered orphan"))).toBe(true); + expect(seen.some((evt) => evt.kind === "capture.failed")).toBe(false); + expect(seen.some((evt) => evt.kind === "capture.done")).toBe(true); + expect(tmp.repos.traces.listAllForEpisode("ep_1" as EpisodeId)).toHaveLength(1); }); it("caps reflect-phase LLM calls per episode", async () => { diff --git a/apps/memos-local-plugin/tests/unit/logger/signal-ownership.test.ts b/apps/memos-local-plugin/tests/unit/logger/signal-ownership.test.ts new file mode 100644 index 000000000..9e34440c4 --- /dev/null +++ b/apps/memos-local-plugin/tests/unit/logger/signal-ownership.test.ts @@ -0,0 +1,23 @@ +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; + +import { describe, expect, it } from "vitest"; + +describe("process signal ownership", () => { + it("keeps SIGINT/SIGTERM out of the shared logger", () => { + const source = readFileSync(resolve("core/logger/index.ts"), "utf8"); + + expect(source).not.toMatch(/process\.once\("SIG(?:INT|TERM)"/); + expect(source).not.toMatch(/process\.exit\(13[0-9]\)|process\.exit\(14[0-9]\)/); + expect(source).toMatch(/process\.once\("beforeExit"/); + }); + + it("leaves graceful signal handling with both executable bridge entries", () => { + for (const entry of ["bridge.cts", "bridge.mts"]) { + const source = readFileSync(resolve(entry), "utf8"); + expect(source).toMatch(/process\.on\("SIGINT"/); + expect(source).toMatch(/process\.on\("SIGTERM"/); + expect(source).toMatch(/await (?:withShutdownTimeout\()?core\.shutdown\(\)/); + } + }); +}); diff --git a/apps/memos-local-plugin/tests/unit/pipeline/auto-recovery-role.test.ts b/apps/memos-local-plugin/tests/unit/pipeline/auto-recovery-role.test.ts new file mode 100644 index 000000000..0988a67f4 --- /dev/null +++ b/apps/memos-local-plugin/tests/unit/pipeline/auto-recovery-role.test.ts @@ -0,0 +1,147 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import type { MemoryCore } from "../../../agent-contract/memory-core.js"; +import { DEFAULT_CONFIG } from "../../../core/config/defaults.js"; +import { resolveHome } from "../../../core/config/paths.js"; +import { rootLogger } from "../../../core/logger/index.js"; +import { + createMemoryCore, + createPipeline, + type PipelineDeps, + type PipelineHandle, +} from "../../../core/pipeline/index.js"; +import type { EpisodeId, SessionId } from "../../../core/types.js"; +import { fakeEmbedder } from "../../helpers/fake-embedder.js"; +import { makeTmpDb, type TmpDbHandle } from "../../helpers/tmp-db.js"; + +function fullMemoryConfig(): typeof DEFAULT_CONFIG { + return { + ...DEFAULT_CONFIG, + algorithm: { + ...DEFAULT_CONFIG.algorithm, + lightweightMemory: { + ...DEFAULT_CONFIG.algorithm.lightweightMemory, + enabled: false, + }, + }, + }; +} + +function buildDeps(db: TmpDbHandle): PipelineDeps { + return { + agent: "hermes", + home: resolveHome("hermes", db.dir), + config: fullMemoryConfig(), + db: db.db, + repos: db.repos, + llm: null, + reflectLlm: null, + embedder: fakeEmbedder({ dimensions: 384 }), + log: rootLogger.child({ channel: "test.auto-recovery-role" }), + namespace: { agentKind: "hermes", profileId: "default" }, + now: () => 1_700_000_000_000, + }; +} + +describe("automatic recovery role ownership", () => { + let db: TmpDbHandle | null = null; + let pipeline: PipelineHandle | null = null; + let core: MemoryCore | null = null; + + afterEach(async () => { + if (core) { + await core.shutdown(); + } else if (pipeline) { + await pipeline.shutdown("test.cleanup"); + } + db?.cleanup(); + core = null; + pipeline = null; + db = null; + vi.restoreAllMocks(); + }); + + it("leaves startup rows untouched but keeps normal turn capture enabled", async () => { + db = makeTmpDb({ agent: "hermes" }); + const orphanSessionId = "se_viewer_orphan" as SessionId; + const orphanEpisodeId = "ep_viewer_orphan" as EpisodeId; + db.repos.sessions.upsert({ + id: orphanSessionId, + agent: "hermes", + ownerAgentKind: "hermes", + ownerProfileId: "default", + ownerWorkspaceId: null, + startedAt: 1_600_000_000_000, + lastSeenAt: 1_600_000_000_000, + meta: {}, + }); + db.repos.episodes.insert({ + id: orphanEpisodeId, + sessionId: orphanSessionId, + ownerAgentKind: "hermes", + ownerProfileId: "default", + ownerWorkspaceId: null, + startedAt: 1_600_000_000_000, + endedAt: null, + traceIds: [], + rTask: 0.7, + status: "open", + meta: { sentinel: "unchanged" }, + }); + + const intervalSpy = vi.spyOn(globalThis, "setInterval"); + pipeline = createPipeline(buildDeps(db)); + core = createMemoryCore( + pipeline, + resolveHome("hermes", db.dir), + "auto-recovery-role-test", + { autoRecovery: false }, + ); + await core.init(); + await core.waitForStartupRecovery?.(); + + const untouched = db.repos.episodes.getById(orphanEpisodeId); + expect(untouched?.status).toBe("open"); + expect(untouched?.endedAt).toBeNull(); + expect(untouched?.meta).toEqual({ sentinel: "unchanged" }); + expect( + intervalSpy.mock.calls.some(([, delay]) => delay === 10 * 60 * 1000), + ).toBe(false); + + const sessionId = await core.openSession({ + agent: "hermes", + sessionId: "se_normal_capture" as SessionId, + }); + const start = await core.onTurnStart({ + agent: "hermes", + sessionId, + userText: "我喜欢的水果是凤梨", + ts: 1_700_000_000_100, + }); + const saved = await core.onTurnEnd({ + agent: "hermes", + sessionId, + episodeId: start.query.episodeId!, + agentText: "记住了。", + toolCalls: [], + ts: 1_700_000_000_200, + }); + await pipeline.flush(); + + const trace = await core.getTrace(saved.traceId); + expect(trace?.userText).toBe("我喜欢的水果是凤梨"); + }); + + it("uses the same Hermes-only role formula in both bridge entries", () => { + const root = join(__dirname, "../../.."); + for (const filename of ["bridge.mts", "bridge.cts"]) { + const source = readFileSync(join(root, filename), "utf8"); + expect(source).toContain( + 'autoRecovery: args.agent !== "hermes" || !args.daemon', + ); + } + }); +}); diff --git a/apps/memos-local-plugin/tests/unit/pipeline/memory-core.test.ts b/apps/memos-local-plugin/tests/unit/pipeline/memory-core.test.ts index aad881593..cb5140d09 100644 --- a/apps/memos-local-plugin/tests/unit/pipeline/memory-core.test.ts +++ b/apps/memos-local-plugin/tests/unit/pipeline/memory-core.test.ts @@ -1549,7 +1549,7 @@ algorithm: expect(meta.reward?.traceIds).toEqual(["tr_dirty"]); }); - it("rescoring finalized closed episodes that have traces but no reward metadata", async () => { + it("recovers a real SQLite dirty episode when trace ts differs from turnId", async () => { home = await makeTmpHome({ agent: "openclaw", configYaml: FULL_MEMORY_CONFIG_YAML, @@ -1567,6 +1567,7 @@ algorithm: const Sqlite = (await import("better-sqlite3")).default; const writeDb = new Sqlite(home.home.dbFile); const ts = Date.now() - 1_000; + const turnId = ts - 250; writeDb .prepare( `INSERT INTO sessions (id, agent, started_at, last_seen_at, meta_json) VALUES (?, ?, ?, ?, ?)`, @@ -1611,7 +1612,7 @@ algorithm: 0.5, "[]", "[]", - ts, + turnId, 1, ); writeDb.close(); @@ -1826,6 +1827,177 @@ algorithm: await core.waitForStartupRecovery?.(); }); + it("does not run the stale-open manual reward fallback after capture.failed", async () => { + const startedAt = 1_000; + db!.repos.sessions.upsert({ + id: "se_capture_failed", + agent: "openclaw", + ownerAgentKind: "openclaw", + ownerProfileId: "main", + ownerWorkspaceId: null, + startedAt, + lastSeenAt: startedAt, + meta: {}, + }); + db!.repos.episodes.insert({ + id: "ep_capture_failed", + sessionId: "se_capture_failed", + ownerAgentKind: "openclaw", + ownerProfileId: "main", + ownerWorkspaceId: null, + startedAt, + endedAt: null, + traceIds: ["tr_capture_failed"] as never, + rTask: null, + status: "open", + meta: {}, + }); + db!.repos.traces.insert({ + id: "tr_capture_failed", + episodeId: "ep_capture_failed", + sessionId: "se_capture_failed", + ownerAgentKind: "openclaw", + ownerProfileId: "main", + ownerWorkspaceId: null, + ts: startedAt + 100, + userText: "recover this stale episode", + agentText: "the capture stage will fail in this test", + summary: "stale capture failure", + toolCalls: [], + reflection: null, + agentThinking: null, + value: 0, + alpha: 0, + rHuman: null, + priority: 0.5, + tags: [], + errorSignatures: [], + vecSummary: null, + vecAction: null, + share: null, + turnId: startedAt, + schemaVersion: 1, + }); + + pipeline = createPipeline(buildDeps(db!)); + let captureCalls = 0; + let rewardCalls = 0; + pipeline.captureRunner.runReflect = async (input) => { + captureCalls++; + pipeline!.buses.capture.emit({ + kind: "capture.failed", + episodeId: input.episode.id, + sessionId: input.episode.sessionId, + stage: "match", + error: { code: "conflict", message: "injected full replay mismatch" }, + }); + throw new Error("injected full replay mismatch"); + }; + const originalRewardRun = pipeline.rewardRunner.run.bind(pipeline.rewardRunner); + pipeline.rewardRunner.run = async (input) => { + rewardCalls++; + return originalRewardRun(input); + }; + + core = createMemoryCore( + pipeline, + resolveHome("openclaw", "/tmp/memos-mc-test"), + "capture-failure-recovery", + ); + await core.init(); + await core.waitForStartupRecovery?.(); + + expect(captureCalls).toBe(1); + expect(rewardCalls).toBe(0); + expect(db!.repos.episodes.getById("ep_capture_failed" as never)?.rTask).toBeNull(); + }); + + it("cleans a legacy dirty marker when reward was intentionally skipped", async () => { + const startedAt = 1_000; + db!.repos.sessions.upsert({ + id: "se_skipped_dirty", + agent: "openclaw", + ownerAgentKind: "openclaw", + ownerProfileId: "main", + ownerWorkspaceId: null, + startedAt, + lastSeenAt: startedAt, + meta: {}, + }); + db!.repos.episodes.insert({ + id: "ep_skipped_dirty", + sessionId: "se_skipped_dirty", + ownerAgentKind: "openclaw", + ownerProfileId: "main", + ownerWorkspaceId: null, + startedAt, + endedAt: startedAt + 100, + traceIds: ["tr_skipped_dirty"] as never, + rTask: null, + status: "closed", + meta: { + closeReason: "finalized", + reward: { + source: "heuristic", + reason: "对话内容过短", + scoredAt: startedAt + 100, + trigger: "implicit_fallback", + skipped: true, + }, + rewardDirty: { + failedAttempts: 7, + lastFailureAt: startedAt + 100, + }, + }, + }); + db!.repos.traces.insert({ + id: "tr_skipped_dirty", + episodeId: "ep_skipped_dirty", + sessionId: "se_skipped_dirty", + ownerAgentKind: "openclaw", + ownerProfileId: "main", + ownerWorkspaceId: null, + ts: startedAt + 100, + userText: "我喜欢玩的游戏呢", + agentText: "你喜欢马里奥。", + summary: null, + toolCalls: [], + reflection: null, + agentThinking: null, + value: 0, + alpha: 0, + rHuman: null, + priority: 0.5, + tags: [], + errorSignatures: [], + vecSummary: null, + vecAction: null, + share: null, + turnId: startedAt, + schemaVersion: 1, + }); + + pipeline = createPipeline(buildDeps(db!)); + let captureCalls = 0; + pipeline.captureRunner.runReflect = async (input) => { + captureCalls++; + throw new Error(`unexpected recovery for ${input.episode.id}`); + }; + core = createMemoryCore( + pipeline, + resolveHome("openclaw", "/tmp/memos-mc-test"), + "skipped-dirty-cleanup", + ); + + await core.init(); + await core.waitForStartupRecovery?.(); + + expect(captureCalls).toBe(0); + const episode = db!.repos.episodes.getById("ep_skipped_dirty" as never); + expect(episode?.meta?.reward).toMatchObject({ skipped: true }); + expect(episode?.meta?.rewardDirty).toBeUndefined(); + }); + it("dirty closed episodes hit a failure-count backoff so init() does not retry them every restart (issue #1808)", async () => { // The OpenClaw report noted "orphan episodes with failed LLM calls // are retried indefinitely with no backoff". After the third diff --git a/apps/memos-local-plugin/tests/unit/reward/reward.integration.test.ts b/apps/memos-local-plugin/tests/unit/reward/reward.integration.test.ts index 238713a29..568874020 100644 --- a/apps/memos-local-plugin/tests/unit/reward/reward.integration.test.ts +++ b/apps/memos-local-plugin/tests/unit/reward/reward.integration.test.ts @@ -284,6 +284,70 @@ describe("reward/integration", () => { expect(t.priority).toBe(0); }); + it("treats a triviality skip as terminal and clears an inherited dirty marker", async () => { + const sid = "s_int_skipped"; + const eid = "ep_int_skipped"; + seedEpisode(handle, eid, sid, ["tr_short"]); + seedTrace(handle, "tr_short", eid, sid, { + userText: "我喜欢玩的游戏呢", + agentText: "你喜欢马里奥。", + }); + handle.repos.episodes.updateMeta(eid as unknown as EpisodeRow["id"], { + closeReason: "finalized", + rewardDirty: { + failedAttempts: 7, + lastFailureAt: NOW - 1_000, + }, + }); + + const shortSnapshot: EpisodeSnapshot = { + ...rewardSnapshot(eid, sid, ["tr_short"]), + turnCount: 1, + turns: [ + { + role: "user", + content: "我喜欢玩的游戏呢", + ts: NOW, + meta: {}, + }, + { + role: "assistant", + content: "你喜欢马里奥。", + ts: NOW, + meta: {}, + }, + ], + }; + const runner = createRewardRunner({ + tracesRepo: handle.repos.traces, + episodesRepo: handle.repos.episodes, + feedbackRepo: handle.repos.feedback, + getEpisodeSnapshot: () => shortSnapshot, + llm: null, + bus: createRewardEventBus(), + cfg: { + ...cfg(), + minExchangesForCompletion: 2, + minContentCharsForCompletion: 80, + }, + now: () => NOW, + }); + + const result = await runner.run({ + episodeId: eid as unknown as Parameters[0]["episodeId"], + feedback: [], + trigger: "implicit_fallback", + }); + + expect(result.rHuman).toBe(0); + const episode = handle.repos.episodes.getById( + eid as unknown as EpisodeRow["id"], + )!; + expect(episode.rTask).toBeNull(); + expect(episode.meta?.reward).toMatchObject({ skipped: true }); + expect(episode.meta?.rewardDirty).toBeUndefined(); + }); + it("episodes with no traces still score R_human but skip backprop", async () => { const sid = "s_int_3"; const eid = "ep_int_3"; diff --git a/apps/memos-local-plugin/tests/unit/server/admin.test.ts b/apps/memos-local-plugin/tests/unit/server/admin.test.ts index 6eb61c543..dc026650e 100644 --- a/apps/memos-local-plugin/tests/unit/server/admin.test.ts +++ b/apps/memos-local-plugin/tests/unit/server/admin.test.ts @@ -3,7 +3,10 @@ import { EventEmitter } from "node:events"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { MemoryCore } from "../../../agent-contract/memory-core.js"; -import { registerAdminRoutes } from "../../../server/routes/admin.js"; +import { + isSupervisorManagedProcess, + registerAdminRoutes, +} from "../../../server/routes/admin.js"; import { Routes } from "../../../server/routes/registry.js"; const { spawnMock } = vi.hoisted(() => ({ @@ -33,12 +36,16 @@ describe("admin lifecycle routes", () => { vi.useRealTimers(); }); - it("replaces the Hermes viewer daemon so saved model config takes effect", async () => { + it("lets the supervisor replace a managed Hermes viewer", async () => { + const requestShutdown = vi.fn(); const routes = new Routes(); registerAdminRoutes( routes, { core: {} as MemoryCore }, - { agent: "hermes" }, + { + agent: "hermes", + lifecycle: { supervised: true, requestShutdown }, + }, ); const restart = routes.getExact("POST /api/v1/admin/restart"); @@ -46,6 +53,34 @@ describe("admin lifecycle routes", () => { const result = await restart!({} as never); + expect(result).toMatchObject({ ok: true, restarting: true }); + expect(spawnMock).not.toHaveBeenCalledWith( + "bash", + expect.anything(), + expect.anything(), + ); + expect(requestShutdown).not.toHaveBeenCalled(); + + await vi.advanceTimersByTimeAsync(200); + + expect(requestShutdown).toHaveBeenCalledOnce(); + }); + + it("retains the detached replacement fallback for a portable Hermes viewer", async () => { + const requestShutdown = vi.fn(); + const routes = new Routes(); + registerAdminRoutes( + routes, + { core: {} as MemoryCore }, + { + agent: "hermes", + lifecycle: { supervised: false, requestShutdown }, + }, + ); + + const restart = routes.getExact("POST /api/v1/admin/restart"); + const result = await restart!({} as never); + expect(result).toMatchObject({ ok: true, restarting: true }); expect(spawnMock).toHaveBeenCalledWith( "bash", @@ -58,5 +93,18 @@ describe("admin lifecycle routes", () => { stdio: "ignore", }), ); + + await vi.advanceTimersByTimeAsync(200); + expect(requestShutdown).toHaveBeenCalledOnce(); + }); + + it("recognises launchd and systemd without treating the macOS shell sentinel as supervised", () => { + expect(isSupervisorManagedProcess({ XPC_SERVICE_NAME: "ai.memtensor.memos-local-hermes" })).toBe(true); + expect(isSupervisorManagedProcess({ XPC_SERVICE_NAME: "ai.memtensor.memos-local-hermes.nova" })).toBe(true); + expect(isSupervisorManagedProcess({ INVOCATION_ID: "abc123" })).toBe(true); + expect(isSupervisorManagedProcess({ XPC_SERVICE_NAME: "application.com.example.desktop" })).toBe(false); + expect(isSupervisorManagedProcess({ JOURNAL_STREAM: "8:12345" })).toBe(false); + expect(isSupervisorManagedProcess({ XPC_SERVICE_NAME: "0" })).toBe(false); + expect(isSupervisorManagedProcess({})).toBe(false); }); }); From 83b17274a7951413dc07f107dbcc109174cfef6a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 29 Jul 2026 03:58:06 +0000 Subject: [PATCH 07/18] release: @memtensor/memos-local-plugin v2.0.12-beta.1 --- apps/memos-local-plugin/adapters/hermes/plugin.yaml | 2 +- apps/memos-local-plugin/package-lock.json | 4 ++-- apps/memos-local-plugin/package.json | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/memos-local-plugin/adapters/hermes/plugin.yaml b/apps/memos-local-plugin/adapters/hermes/plugin.yaml index 3c1171bea..dd9517d18 100644 --- a/apps/memos-local-plugin/adapters/hermes/plugin.yaml +++ b/apps/memos-local-plugin/adapters/hermes/plugin.yaml @@ -1,5 +1,5 @@ name: memtensor -version: 2.0.11 +version: 2.0.12-beta.1 description: >- MemOS Local — Reflect2Evolve V7 memory for hermes-agent. Layered L1/L2/L3 traces, reflection-weighted reward backprop, diff --git a/apps/memos-local-plugin/package-lock.json b/apps/memos-local-plugin/package-lock.json index 1fdee41b5..46db4c00b 100644 --- a/apps/memos-local-plugin/package-lock.json +++ b/apps/memos-local-plugin/package-lock.json @@ -1,12 +1,12 @@ { "name": "@memtensor/memos-local-plugin", - "version": "2.0.11", + "version": "2.0.12-beta.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@memtensor/memos-local-plugin", - "version": "2.0.11", + "version": "2.0.12-beta.1", "hasInstallScript": true, "license": "MIT", "dependencies": { diff --git a/apps/memos-local-plugin/package.json b/apps/memos-local-plugin/package.json index a4272f568..48e07b94b 100644 --- a/apps/memos-local-plugin/package.json +++ b/apps/memos-local-plugin/package.json @@ -1,6 +1,6 @@ { "name": "@memtensor/memos-local-plugin", - "version": "2.0.11", + "version": "2.0.12-beta.1", "description": "Reflect2Evolve memory plugin: layered L1/L2/L3 memory, reflection-weighted value backprop, cross-task policy induction, skill crystallization, three-tier retrieval. Adapters for OpenClaw and Hermes Agent via a shared algorithm core.", "type": "module", "main": "dist/core/index.js", From 7f1fb4e16e5b6f189b383af0c3ae9c01083a38b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=81=E5=9C=A8=E5=90=B5=E7=9D=80=E5=90=83=E7=B3=96?= Date: Wed, 29 Jul 2026 15:02:45 +0800 Subject: [PATCH 08/18] fix(plugin): satisfy Python lint checks --- .../hermes/memos_provider/__init__.py | 10 +++----- .../hermes/memos_provider/bridge_client.py | 4 +-- .../tests/python/test_bridge_client.py | 9 +++---- .../python/test_hermes_provider_pipeline.py | 2 +- .../python/test_shared_bridge_runtime.py | 25 +++++++++++-------- 5 files changed, 25 insertions(+), 25 deletions(-) diff --git a/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py b/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py index e663b87cb..027f21ba0 100644 --- a/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py +++ b/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py @@ -65,13 +65,13 @@ if str(_PLUGIN_DIR) not in sys.path: sys.path.insert(0, str(_PLUGIN_DIR)) -from bridge_client import BridgeError, MemosBridgeClient -from daemon_manager import ( +from bridge_client import BridgeError, MemosBridgeClient # noqa: E402 +from daemon_manager import ( # noqa: E402 ensure_bridge_running, ensure_viewer_daemon, kill_zombie_bridges, ) -from shared_bridge_runtime import ( +from shared_bridge_runtime import ( # noqa: E402 HERMES_HOOK_DISPATCHER, SHARED_BRIDGE_REGISTRY, SharedBridgeLease, @@ -2138,9 +2138,7 @@ def _reconnect_bridge(self, session_id: str = "", *, timeout: float = 30.0) -> N new_bridge: MemosBridgeClient | None = None try: runtime_home = self._runtime_home or _resolved_memos_runtime_home() - runtime_env = dict( - self._runtime_env or _memos_runtime_env_snapshot(runtime_home) - ) + runtime_env = dict(self._runtime_env or _memos_runtime_env_snapshot(runtime_home)) new_bridge = MemosBridgeClient( runtime_home=str(runtime_home), extra_env=runtime_env, diff --git a/apps/memos-local-plugin/adapters/hermes/memos_provider/bridge_client.py b/apps/memos-local-plugin/adapters/hermes/memos_provider/bridge_client.py index 841f4b687..6863ba1ef 100644 --- a/apps/memos-local-plugin/adapters/hermes/memos_provider/bridge_client.py +++ b/apps/memos-local-plugin/adapters/hermes/memos_provider/bridge_client.py @@ -51,7 +51,7 @@ def _expanded_path(value: str, env: dict[str, str]) -> Path: home = env.get("HOME", "").strip() or str(Path.home()) if value == "~": value = home - elif value.startswith("~/") or value.startswith("~\\"): + elif value.startswith(("~/", "~\\")): value = str(Path(home) / value[2:]) return Path(value).resolve() @@ -71,7 +71,7 @@ def _resolved_runtime_home(agent: str, env: dict[str, str]) -> Path: def _runtime_scope_token(agent: str, runtime_home: Path) -> str: """Return a stable, path-private token safe for use in a PID filename.""" - raw = f"{agent}\0{runtime_home}".encode("utf-8") + raw = f"{agent}\0{runtime_home}".encode() return hashlib.sha256(raw).hexdigest()[:24] diff --git a/apps/memos-local-plugin/tests/python/test_bridge_client.py b/apps/memos-local-plugin/tests/python/test_bridge_client.py index 488632e85..82763fcea 100644 --- a/apps/memos-local-plugin/tests/python/test_bridge_client.py +++ b/apps/memos-local-plugin/tests/python/test_bridge_client.py @@ -29,10 +29,10 @@ if str(_p) not in sys.path: sys.path.insert(0, str(_p)) -import bridge_client as bridge_client_mod -import daemon_manager as daemon_manager_mod +import bridge_client as bridge_client_mod # noqa: E402 +import daemon_manager as daemon_manager_mod # noqa: E402 -from bridge_client import BridgeError, MemosBridgeClient +from bridge_client import BridgeError, MemosBridgeClient # noqa: E402 class FakePopen: @@ -395,8 +395,7 @@ def test_stdio_bridge_passes_stable_runtime_scope(self) -> None: ) assert self._fake is not None scope_args = [ - arg for arg in getattr(self._fake, "cmd", []) - if arg.startswith("--runtime-scope=") + arg for arg in getattr(self._fake, "cmd", []) if arg.startswith("--runtime-scope=") ] self.assertEqual(len(scope_args), 1) token = scope_args[0].split("=", 1)[1] diff --git a/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py b/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py index 02b78b307..7ecaccb89 100644 --- a/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py +++ b/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py @@ -22,7 +22,7 @@ if str(_p) not in sys.path: sys.path.insert(0, str(_p)) -import memos_provider +import memos_provider # noqa: E402 class FakeBridge: diff --git a/apps/memos-local-plugin/tests/python/test_shared_bridge_runtime.py b/apps/memos-local-plugin/tests/python/test_shared_bridge_runtime.py index 8247cbbd9..1cbc95ae8 100644 --- a/apps/memos-local-plugin/tests/python/test_shared_bridge_runtime.py +++ b/apps/memos-local-plugin/tests/python/test_shared_bridge_runtime.py @@ -16,10 +16,10 @@ if str(_p) not in sys.path: sys.path.insert(0, str(_p)) -import memos_provider +import memos_provider # noqa: E402 -from bridge_client import BridgeError -from shared_bridge_runtime import ( +from bridge_client import BridgeError # noqa: E402 +from shared_bridge_runtime import ( # noqa: E402 HermesHookDispatcher, SharedBridgeRuntimeRegistry, ) @@ -154,8 +154,12 @@ def test_host_handler_is_registered_once_per_bridge_generation(self) -> None: lease_a = self.registry.acquire(("shared-home",), client_factory=lambda: bridge) lease_b = self.registry.acquire(("shared-home",), client_factory=lambda: bridge) - handler_a = lambda _params: {"content": "a"} - handler_b = lambda _params: {"content": "b"} + def handler_a(_params: object) -> dict[str, str]: + return {"content": "a"} + + def handler_b(_params: object) -> dict[str, str]: + return {"content": "b"} + lease_a.register_host_handler("host.llm.complete", handler_a) lease_b.register_host_handler("host.llm.complete", handler_b) @@ -179,7 +183,10 @@ def request( return super().request(method, params, **kwargs) self_test = self - handler = lambda _params: {"content": "ready"} + + def handler(_params: object) -> dict[str, str]: + return {"content": "ready"} + bridge = StartupBridge() lease = self.registry.acquire( ("shared-home",), @@ -456,11 +463,7 @@ def request( return super().request(method, params, **kwargs) def factory(*_args: object, **_kwargs: object) -> FakeBridge: - bridge: FakeBridge - if not created: - bridge = FirstSessionOpenFails() - else: - bridge = FakeBridge() + bridge = FirstSessionOpenFails() if not created else FakeBridge() created.append(bridge) return bridge From 5cdab0578b12ed43d364c1890e02e495bfb13cbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=81=E5=9C=A8=E5=90=B5=E7=9D=80=E5=90=83=E7=B3=96?= Date: Fri, 31 Jul 2026 18:03:09 +0800 Subject: [PATCH 09/18] fix(plugin): improve retrieval relevance ranking --- apps/memos-local-plugin/agent-contract/dto.ts | 15 + .../core/injection/scheduler.ts | 14 + .../core/pipeline/memory-core.ts | 65 ++- .../core/pipeline/orchestrator.ts | 1 + .../core/pipeline/retrieval-log.ts | 64 +++ .../core/pipeline/retrieval-repos.ts | 2 + .../core/retrieval/candidate-dedupe.ts | 135 ++++++ .../core/retrieval/injector.ts | 11 + .../core/retrieval/llm-filter.ts | 147 +----- .../core/retrieval/profile.ts | 28 ++ .../core/retrieval/query-builder.ts | 40 +- .../core/retrieval/ranker.ts | 237 ++++++++- .../core/retrieval/retrieve.ts | 181 +++++-- .../core/retrieval/tier1-skill.ts | 35 +- .../core/retrieval/tier2-experience.ts | 33 +- .../core/retrieval/tier2-trace.ts | 152 ++++-- .../core/retrieval/tier3-world.ts | 38 +- .../core/retrieval/types.ts | 19 + .../core/storage/keyword.ts | 39 +- .../core/storage/repos/policies.ts | 22 +- .../core/storage/repos/skills.ts | 22 +- .../core/storage/repos/traces.ts | 23 +- .../core/storage/repos/world_model.ts | 22 +- .../tests/unit/injection/scheduler.test.ts | 48 +- .../tests/unit/pipeline/retrieval-log.test.ts | 61 +++ .../tests/unit/retrieval/integration.test.ts | 159 +++++- .../tests/unit/retrieval/llm-filter.test.ts | 116 +---- .../unit/retrieval/query-builder.test.ts | 57 ++- .../tests/unit/retrieval/ranker.test.ts | 456 +++++++++++++++++- .../tests/unit/retrieval/tier1.test.ts | 19 + .../tests/unit/retrieval/tier2.test.ts | 176 +++++++ .../tests/unit/retrieval/tier3.test.ts | 5 +- .../tests/unit/storage/fts-keyword.test.ts | 27 +- .../tests/unit/storage/keyword.test.ts | 22 +- .../viewer/src/views/LogsView.tsx | 29 ++ 35 files changed, 2030 insertions(+), 490 deletions(-) create mode 100644 apps/memos-local-plugin/core/pipeline/retrieval-log.ts create mode 100644 apps/memos-local-plugin/core/retrieval/candidate-dedupe.ts create mode 100644 apps/memos-local-plugin/core/retrieval/profile.ts create mode 100644 apps/memos-local-plugin/tests/unit/pipeline/retrieval-log.test.ts diff --git a/apps/memos-local-plugin/agent-contract/dto.ts b/apps/memos-local-plugin/agent-contract/dto.ts index bdb9375ab..e9ae71101 100644 --- a/apps/memos-local-plugin/agent-contract/dto.ts +++ b/apps/memos-local-plugin/agent-contract/dto.ts @@ -644,6 +644,21 @@ export interface InjectionSnippet { title?: string; body: string; score?: number; + /** Structured score composition for retrieval logs and diagnostics. */ + scoreDetails?: InjectionScoreDetails; +} + +export interface InjectionScoreDetails { + profile: string; + semantic: number; + tierBoost: number; + rrfBoost: number; + relevance: number; + mmrLambda: number; + redundancy: number; + finalScore: number; + channels: string[]; + bypassedThreshold: boolean; } /** diff --git a/apps/memos-local-plugin/core/injection/scheduler.ts b/apps/memos-local-plugin/core/injection/scheduler.ts index 4cfde07f8..615cdebdb 100644 --- a/apps/memos-local-plugin/core/injection/scheduler.ts +++ b/apps/memos-local-plugin/core/injection/scheduler.ts @@ -1,4 +1,6 @@ import type { SessionId, EpisodeId } from "../../agent-contract/dto.js"; +import { classifyRetrievalProfile } from "../retrieval/profile.js"; +import type { RetrievalProfile } from "../retrieval/types.js"; import type { IntentDecision, TurnRelation } from "../session/types.js"; export type InjectionScenarioId = @@ -20,6 +22,7 @@ export interface SchedulerContext { export interface RetrievePlan { scenarioId: InjectionScenarioId; + profile: RetrievalProfile; entry: "turn_start" | "turn_start_skip"; wantTier1: boolean; wantTier2: boolean; @@ -43,6 +46,14 @@ export function scheduleInjection(ctx: SchedulerContext): RetrievePlan { } if (intent.kind === "memory_probe") { + const profile = classifyRetrievalProfile(ctx.userText); + if (profile === "personal_fact") { + return retrievePlan( + "MEMORY_PROBE", + { tier1: false, tier2: true, tier3: false }, + profile, + ); + } return retrievePlan("MEMORY_PROBE", intent.retrieval); } @@ -64,6 +75,7 @@ export function scheduleInjection(ctx: SchedulerContext): RetrievePlan { function skipPlan(scenarioId: Extract): RetrievePlan { return { scenarioId, + profile: "default", entry: "turn_start_skip", wantTier1: false, wantTier2: false, @@ -75,9 +87,11 @@ function skipPlan(scenarioId: Extract) function retrievePlan( scenarioId: Exclude, retrieval: IntentDecision["retrieval"], + profile: RetrievalProfile = "default", ): RetrievePlan { return { scenarioId, + profile, entry: "turn_start", wantTier1: retrieval.tier1, wantTier2: retrieval.tier2, diff --git a/apps/memos-local-plugin/core/pipeline/memory-core.ts b/apps/memos-local-plugin/core/pipeline/memory-core.ts index ddcdf1602..c9254e092 100644 --- a/apps/memos-local-plugin/core/pipeline/memory-core.ts +++ b/apps/memos-local-plugin/core/pipeline/memory-core.ts @@ -99,6 +99,10 @@ import type { ReasoningConfig } from "../llm/types.js"; import { createPipeline } from "./orchestrator.js"; import { RECOVERY_REASONS } from "./recovery-constants.js"; import { wrapRetrievalRepos } from "./retrieval-repos.js"; +import { + buildLocalRetrievalLogStages, + type RetrievalLogCandidate, +} from "./retrieval-log.js"; import type { PipelineDeps, PipelineHandle } from "./types.js"; import { namespaceFromHints, @@ -975,12 +979,9 @@ export function createMemoryCore( return text.split(/\n+/).map((line) => line.trim()).find(Boolean)?.slice(0, 240) ?? ""; } - function logCandidatesFromHits(hits: readonly RetrievalHitDTO[]): Array<{ - tier: number; - refKind: string; - refId: string; - score: number; - snippet: string; + function logCandidatesFromHits( + hits: readonly RetrievalHitDTO[], + ): Array { return hits.map((h) => ({ @@ -2176,25 +2177,13 @@ export function createMemoryCore( // each real agent turn. Without this, `memos_search` rows // only showed up when the viewer's search box was used. try { - const snippets = packet?.snippets ?? []; - const candidates = snippets.map((s) => ({ - tier: inferTier(s.refKind), - refKind: s.refKind, - refId: s.refId, - score: s.score ?? 0, - snippet: s.body, - })); - const droppedIds = new Set( - (packet?.droppedByLlm ?? []).map((s) => s.refId as string), - ); - const localFiltered = candidates.filter((c) => !droppedIds.has(c.refId)); + const localStages = buildLocalRetrievalLogStages(packet); const filtered = hubCandidates.length > 0 ? finalFilteredCandidates - : localFiltered; - const localDropped = candidates.filter((c) => droppedIds.has(c.refId)); + : localStages.filtered; const dropped = hubCandidates.length > 0 - ? [...localDropped, ...finalDroppedCandidates] - : localDropped; + ? [...localStages.dropped, ...finalDroppedCandidates] + : localStages.dropped; const stats = packet ? handle.consumeRetrievalStats(packet.packetId) : null; handle.repos.apiLogs.insert({ toolName: "memos_search", @@ -2207,7 +2196,7 @@ export function createMemoryCore( }, output: ok ? { - candidates, + candidates: localStages.candidates, hubCandidates, filtered, droppedByLlm: dropped, @@ -2787,13 +2776,7 @@ export function createMemoryCore( const ts = Date.now(); const startedAt = Date.now(); let ok = true; - let candidates: Array<{ - tier: number; - refKind: string; - refId: string; - score: number; - snippet: string; - }> = []; + let candidates: RetrievalLogCandidate[] = []; let filtered: typeof candidates = []; let droppedByFinalFilter: typeof candidates = []; let hubCandidates: typeof candidates = []; @@ -2812,6 +2795,8 @@ export function createMemoryCore( args: { ...(query.filters ?? {}), query: query.query }, ts, }, { skipLlmFilter: hubHits.length > 0 }); + const localLogStages = buildLocalRetrievalLogStages(result.packet); + candidates = localLogStages.candidates; let hits: RetrievalHitDTO[] = result.packet.snippets.map((snip) => ({ tier: inferTier(snip.refKind), refId: snip.refId, @@ -2869,15 +2854,11 @@ export function createMemoryCore( // everything tiered/retrieved; `filtered` is what the injector // kept (≤ `maxSnippets`), matching the legacy "LLM filtered" // semantics the user complained about. - candidates = hits.map((h) => ({ - tier: h.tier, - refKind: h.refKind, - refId: h.refId, - score: h.score, - snippet: h.snippet, - })); filtered = logCandidatesFromHits(returnedHits); // final list returned to the adapter. - droppedByFinalFilter = logCandidatesFromHits(final.dropped); + droppedByFinalFilter = [ + ...localLogStages.dropped, + ...logCandidatesFromHits(final.dropped), + ]; // Three-stage observability — surfaced verbatim so the viewer's // Logs page can render "raw → threshold → ranked → LLM filter" @@ -5726,6 +5707,9 @@ type RetrievalStatsLogPayload = { raw?: number; ranked?: number; droppedByThreshold?: number; + dedupedBeforeMmr?: number; + dedupedAfterThreshold?: number; + droppedByKeywordConfirmation?: number; thresholdFloor?: number; topRelevance?: number; llmFilter?: { @@ -5737,6 +5721,7 @@ type RetrievalStatsLogPayload = { channelHits?: Record; queryTokens?: number; queryTags?: string[]; + exactIdentifierCount?: number; embedding?: import("../retrieval/types.js").RetrievalStats["embedding"]; localReturned?: number; hubReturned?: number; @@ -5758,6 +5743,9 @@ function retrievalStatsPayload(s: import("../retrieval/types.js").RetrievalStats raw: s.rawCandidateCount, ranked: s.rankedCount, droppedByThreshold: s.droppedByThresholdCount, + dedupedBeforeMmr: s.dedupedBeforeMmrCount, + dedupedAfterThreshold: s.dedupedAfterThresholdCount, + droppedByKeywordConfirmation: s.droppedByKeywordConfirmationCount, thresholdFloor: s.thresholdFloor, topRelevance: s.topRelevance, llmFilter: { @@ -5769,6 +5757,7 @@ function retrievalStatsPayload(s: import("../retrieval/types.js").RetrievalStats channelHits: s.channelHits as Record | undefined, queryTokens: s.queryTokens, queryTags: s.queryTags, + exactIdentifierCount: s.exactIdentifierCount, embedding: s.embedding, }; } diff --git a/apps/memos-local-plugin/core/pipeline/orchestrator.ts b/apps/memos-local-plugin/core/pipeline/orchestrator.ts index ad6fc3c44..ac48133ad 100644 --- a/apps/memos-local-plugin/core/pipeline/orchestrator.ts +++ b/apps/memos-local-plugin/core/pipeline/orchestrator.ts @@ -1076,6 +1076,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { plan: plan ? { scenarioId: plan.scenarioId, + profile: plan.profile, wantTier1: plan.wantTier1, wantTier2: plan.wantTier2, wantTier3: plan.wantTier3, diff --git a/apps/memos-local-plugin/core/pipeline/retrieval-log.ts b/apps/memos-local-plugin/core/pipeline/retrieval-log.ts new file mode 100644 index 000000000..32a9c0bb2 --- /dev/null +++ b/apps/memos-local-plugin/core/pipeline/retrieval-log.ts @@ -0,0 +1,64 @@ +import type { + InjectionPacket, + InjectionScoreDetails, + InjectionSnippet, +} from "../../agent-contract/dto.js"; + +export interface RetrievalLogCandidate { + tier: 1 | 2 | 3; + refKind: string; + refId: string; + score: number; + snippet: string; + scoreDetails?: InjectionScoreDetails; +} + +export interface LocalRetrievalLogStages { + /** Mechanical/ranked candidates before the local LLM relevance pass. */ + candidates: RetrievalLogCandidate[]; + /** Candidates retained by the local LLM relevance pass. */ + filtered: RetrievalLogCandidate[]; + /** Candidates rejected by the local LLM relevance pass. */ + dropped: RetrievalLogCandidate[]; +} + +/** + * `InjectionPacket.snippets` is already post-LLM-filter. Reconstruct the + * Logs-page funnel from the kept and explicitly surfaced dropped snippets + * instead of trying to remove dropped ids from an already-filtered list. + * The LLM may reorder its kept subset, so restore the mechanical ranker's + * score-descending order only for the complete pre-filter candidate view. + */ +export function buildLocalRetrievalLogStages( + packet: + | Pick + | null + | undefined, +): LocalRetrievalLogStages { + const filtered = (packet?.snippets ?? []).map(toLogCandidate); + const dropped = (packet?.droppedByLlm ?? []).map(toLogCandidate); + return { + candidates: [...filtered, ...dropped].sort( + (left, right) => right.score - left.score, + ), + filtered, + dropped, + }; +} + +function toLogCandidate(snippet: InjectionSnippet): RetrievalLogCandidate { + return { + tier: inferSnippetTier(snippet.refKind), + refKind: snippet.refKind, + refId: snippet.refId, + score: snippet.score ?? 0, + snippet: snippet.body, + scoreDetails: snippet.scoreDetails, + }; +} + +function inferSnippetTier(refKind: InjectionSnippet["refKind"]): 1 | 2 | 3 { + if (refKind === "skill") return 1; + if (refKind === "world-model") return 3; + return 2; +} diff --git a/apps/memos-local-plugin/core/pipeline/retrieval-repos.ts b/apps/memos-local-plugin/core/pipeline/retrieval-repos.ts index 115abf1d4..6eb60ca24 100644 --- a/apps/memos-local-plugin/core/pipeline/retrieval-repos.ts +++ b/apps/memos-local-plugin/core/pipeline/retrieval-repos.ts @@ -35,6 +35,7 @@ export function wrapRetrievalRepos(repos: Repos, namespace: RuntimeNamespace): R procedureJson: row.procedureJson, decisionGuidance: normaliseSkillDecisionGuidance(row.procedureJson), eta: row.eta, + vec: row.vec, sourcePolicyIds: row.sourcePolicyIds, updatedAt: row.updatedAt, }; @@ -104,6 +105,7 @@ export function wrapRetrievalRepos(repos: Repos, namespace: RuntimeNamespace): R title: row.title, body: row.body, policyIds: row.policyIds, + vec: row.vec, }; }, }, diff --git a/apps/memos-local-plugin/core/retrieval/candidate-dedupe.ts b/apps/memos-local-plugin/core/retrieval/candidate-dedupe.ts new file mode 100644 index 000000000..6aad10e06 --- /dev/null +++ b/apps/memos-local-plugin/core/retrieval/candidate-dedupe.ts @@ -0,0 +1,135 @@ +/** + * High-confidence retrieval-candidate dedupe for repeated trace questions. + * + * This stage is intentionally deterministic and embedding-free. It runs after + * relevance/RRF scoring but before the relative threshold and MMR, so duplicate + * high scores cannot inflate the cutoff and removed rows release Top-K slots. + */ + +import type { RankedCandidate } from "./ranker.js"; +import type { TraceCandidate } from "./types.js"; + +const SHORT_ACK_PATTERNS: readonly RegExp[] = [ + /^(ok|okay|sure|got it|noted|understood|alright|will do|copy|copy that|thanks|thank you|✓|✅|👍)[\s.!]*$/i, + /^(记住了|已记住|已经记住|好的|明白|收到|了解|谢谢)[\s。!]*$/, +]; + +export interface CandidateDedupeResult { + kept: RankedCandidate[]; + dropped: RankedCandidate[]; +} + +export interface CandidateDedupeOptions { + /** + * Optional hard-eligibility partition. Candidates with identical content but + * different downstream capabilities remain separate until that capability + * has been evaluated (for example, relative-threshold bypass). + */ + capabilityKey?: (candidate: RankedCandidate) => string; +} + +/** + * Hard-delete only two high-confidence duplicate shapes: + * 1. explicit short acknowledgements when the same question also has + * evidence-bearing candidates; + * 2. candidates whose normalised summary/reflection/non-ack answer evidence + * is identical. + * + * Distinct evidence — including concise answers such as "苹果" and "芒果" — + * remains separate so updates and contradictions still reach MMR and the later + * LLM relevance filter. + */ +export function dedupeRepeatedQuestionCandidates( + ranked: readonly RankedCandidate[], + options: CandidateDedupeOptions = {}, +): CandidateDedupeResult { + const groups = new Map(); + const ungrouped: RankedCandidate[] = []; + + for (const candidate of ranked) { + if (candidate.candidate.refKind !== "trace") { + ungrouped.push(candidate); + continue; + } + const trace = candidate.candidate as TraceCandidate; + const key = normalizeForDedupe(trace.userText ?? ""); + if (!key) { + ungrouped.push(candidate); + continue; + } + const group = groups.get(key) ?? []; + group.push(candidate); + groups.set(key, group); + } + + const selected = new Set(ungrouped); + for (const group of groups.values()) { + const evidenceBearing = group.filter( + (candidate) => !isScaffoldingOnly(candidate), + ); + const eligible = evidenceBearing.length > 0 ? evidenceBearing : group; + const bestByEvidence = new Map(); + for (const candidate of eligible) { + const evidenceSignature = + evidenceBearing.length > 0 + ? candidateEvidenceSignature(candidate) + : "__scaffolding__"; + const signature = [ + evidenceSignature, + options.capabilityKey?.(candidate) ?? "", + ].join("\u0000"); + const current = bestByEvidence.get(signature); + if (!current || isMoreRelevant(candidate, current)) { + bestByEvidence.set(signature, candidate); + } + } + for (const candidate of bestByEvidence.values()) selected.add(candidate); + } + + return { + kept: ranked.filter((candidate) => selected.has(candidate)), + dropped: ranked.filter((candidate) => !selected.has(candidate)), + }; +} + +function isScaffoldingOnly(ranked: RankedCandidate): boolean { + const trace = ranked.candidate as TraceCandidate; + if ((trace.summary?.trim().length ?? 0) > 0) return false; + if ((trace.reflection?.trim().length ?? 0) > 0) return false; + const agent = trace.agentText?.trim() ?? ""; + return agent.length === 0 || isShortAck(agent); +} + +function candidateEvidenceSignature(ranked: RankedCandidate): string { + const trace = ranked.candidate as TraceCandidate; + const agent = trace.agentText?.trim() ?? ""; + return [ + normalizeForDedupe(trace.summary?.trim() ?? ""), + normalizeForDedupe(trace.reflection?.trim() ?? ""), + isShortAck(agent) ? "" : normalizeForDedupe(agent), + ].join("\u0000"); +} + +function isMoreRelevant( + candidate: RankedCandidate, + current: RankedCandidate, +): boolean { + if (candidate.relevance !== current.relevance) { + return candidate.relevance > current.relevance; + } + return candidate.rrf > current.rrf; +} + +function normalizeForDedupe(text: string): string { + return text + .normalize("NFKC") + .toLowerCase() + .trim() + .replace(/\s+/gu, " ") + .replace(/[.!?。!?]+$/gu, "") + .trim(); +} + +function isShortAck(text: string): boolean { + return SHORT_ACK_PATTERNS.some((pattern) => pattern.test(text)); +} diff --git a/apps/memos-local-plugin/core/retrieval/injector.ts b/apps/memos-local-plugin/core/retrieval/injector.ts index a0f10fef5..8a09352c9 100644 --- a/apps/memos-local-plugin/core/retrieval/injector.ts +++ b/apps/memos-local-plugin/core/retrieval/injector.ts @@ -101,6 +101,17 @@ export function toPacket(input: InjectorInput): InjectorResult { }); if (!snippet) continue; snippet.score = round(r.score, 4); + snippet.scoreDetails = r.scoreDetails + ? { + ...r.scoreDetails, + semantic: round(r.scoreDetails.semantic, 4), + tierBoost: round(r.scoreDetails.tierBoost, 4), + rrfBoost: round(r.scoreDetails.rrfBoost, 4), + relevance: round(r.scoreDetails.relevance, 4), + redundancy: round(r.scoreDetails.redundancy, 4), + finalScore: round(r.scoreDetails.finalScore, 4), + } + : undefined; mapping.push({ snippet, tier: r.candidate.tier, diff --git a/apps/memos-local-plugin/core/retrieval/llm-filter.ts b/apps/memos-local-plugin/core/retrieval/llm-filter.ts index 2d1f77180..142bd626d 100644 --- a/apps/memos-local-plugin/core/retrieval/llm-filter.ts +++ b/apps/memos-local-plugin/core/retrieval/llm-filter.ts @@ -24,35 +24,19 @@ import type { LlmClient } from "../llm/index.js"; import type { Logger } from "../logger/types.js"; import { RETRIEVAL_FILTER_PROMPT } from "../llm/prompts/index.js"; import type { RankedCandidate } from "./ranker.js"; -import type { RetrievalConfig, TraceCandidate } from "./types.js"; +import type { + RetrievalConfig, + RetrievalProfile, +} from "./types.js"; const DEFAULT_CANDIDATE_BODY_CHARS = 500; const MIN_FILTER_OUTPUT_TOKENS = 160; const MAX_FILTER_OUTPUT_TOKENS = 2048; -/** - * A trace whose `agentText` falls under this length, with no LLM summary - * or reflection to back it up, is treated as a near-duplicate question - * trace (issue #1913). The rescue path keeps these *behind* informative - * candidates so the answer-bearing trace surfaces first. - */ -const INFORMATIVE_AGENT_TEXT_MIN_CHARS = 20; - -/** - * Short acknowledgement / scaffold replies that the filter prompt - * rightly classes as "scaffolding chatter". When the LLM filter empties - * the kept set we still need to make a rescue call — these strings let - * us prefer informative replies over plain acks. Bounded list, exact - * matches only after trimming surrounding punctuation / whitespace. - */ -const SHORT_ACK_PATTERNS: readonly RegExp[] = [ - /^(ok|okay|sure|got it|noted|understood|alright|will do|copy|copy that|thanks|thank you|✓|✅|👍)[\s.!]*$/i, - /^(记住了|已记住|已经记住|好的|明白|收到|了解|谢谢)[\s。!]*$/, -]; - export interface FilterInput { query: string; ranked: readonly RankedCandidate[]; + profile?: RetrievalProfile; /** * Episode this retrieval is happening for (typically the active or * just-opening episode). Forwarded to the LLM call so the resulting @@ -90,11 +74,9 @@ export interface FilterResult { | "deferred_to_final" | "llm_kept_all" | "llm_filtered" - // The LLM returned an empty selection over a non-empty ranked list - // (issue #1913 — repeated question traces crowding the hit set). - // We rescued the top-K best-scoring candidates so the agent always - // sees a packet when retrieval succeeded; `sufficient` is forced - // to `false` so downstream callers know the injection is weak. + | "llm_filtered_empty" + // Legacy outcome retained for historical api_logs compatibility. + // New calls no longer mechanically refill a valid empty LLM result. | "llm_filtered_refilled" // The LLM was supposed to run but the call failed / parsed badly. // We applied a mechanical relevance cutoff (top-K above @@ -195,24 +177,16 @@ ${list}`, ); const keepIndices = new Set(cappedIndices); if (keepIndices.size === 0) { - // Issue #1913: the model asked us to drop everything. Honouring - // that verbatim used to collapse `turn.start` injection to "" even - // when retrieval was healthy — the failure mode is a hit set - // dominated by near-duplicate question traces from prior - // sessions, where each candidate individually looks like - // "surface-similar wrong sub-problem" to the filter prompt. - // Instead, rescue the top-K best-scoring candidates (preferring - // informative traces over pure-question / ack-only chatter) so - // the agent always sees a packet when retrieval succeeded. - // `safeCutoff`'s sibling escape hatch (`llmFilterMaxKeep === 0`) - // is honoured so operators can still ask for hard drop. - return rescueFromEmptySelection(ranked, deps, sufficient); + return { + kept: [], + dropped: [...ranked], + outcome: "llm_filtered_empty", + sufficient: sufficient ?? false, + }; } const kept = cappedIndices.map((i) => ranked[i]!); - const dropped: RankedCandidate[] = []; - ranked.forEach((r, i) => { - if (!keepIndices.has(i)) dropped.push(r); - }); + const keptSet = new Set(kept); + const dropped = ranked.filter((candidate) => !keptSet.has(candidate)); return { kept, dropped, @@ -243,95 +217,6 @@ function passthrough( return { kept: [...ranked], dropped: [], outcome, sufficient: null }; } -/** - * Issue #1913 rescue path. Invoked when the LLM relevance filter - * returned `selected: []` for a *non-empty* ranked candidate list — the - * most common cause is a hit set dominated by near-duplicate question - * traces from previous sessions, where the filter prompt's "drop - * scaffolding chatter" / "drop surface-similar wrong sub-problem" - * rubric is applied to every candidate. - * - * Strategy: keep the top-K best-scoring candidates, preferring - * informative traces (skill / episode / experience / world-model, or a - * trace whose `agentText`/`summary`/`reflection` carries real content) - * over pure-question chatter. We do NOT re-query the LLM — the rescue - * is a single O(n) partition + slice. Outcome label is - * `"llm_filtered_refilled"` so the Logs viewer can show "LLM collapsed, - * safety net fired" distinct from a normal `"llm_filtered"`. - * - * Escape hatch: `llmFilterMaxKeep === 0` skips the rescue entirely and - * honours the "drop everything" request (matches existing `safeCutoff` - * semantics for the same config value). - */ -function rescueFromEmptySelection( - ranked: readonly RankedCandidate[], - deps: FilterDeps, - sufficient: boolean | null, -): FilterResult { - const keepCap = Math.max(0, deps.config.llmFilterMaxKeep); - if (keepCap === 0 || ranked.length === 0) { - return { - kept: [], - dropped: [...ranked], - outcome: "llm_filtered", - sufficient: sufficient ?? false, - }; - } - const informative: RankedCandidate[] = []; - const chatter: RankedCandidate[] = []; - for (const r of ranked) { - if (isInformativeCandidate(r)) informative.push(r); - else chatter.push(r); - } - // Preserve ranker order within each bucket; informative first so the - // answer-bearing trace surfaces even when the ranker placed it below - // surface-similar question traces. - const ordered = [...informative, ...chatter]; - const kept = ordered.slice(0, Math.min(keepCap, ordered.length)); - const keptSet = new Set(kept); - const dropped = ranked.filter((r) => !keptSet.has(r)); - deps.log.debug("llm_filter.collapsed_refill", { - ranked: ranked.length, - rescued: kept.length, - informative: informative.length, - chatter: chatter.length, - filteredAll: true, - }); - return { - kept, - dropped, - outcome: "llm_filtered_refilled", - sufficient: sufficient ?? false, - }; -} - -/** - * Returns true when a ranked candidate carries content the agent can - * actually use. Skills, episodes, experiences, and world-models always - * count. Traces count when their `summary` or `reflection` is non-empty - * or their `agentText` is longer than a short acknowledgement. - * - * Used by the rescue path (and intentionally only there) to bias the - * rescued set toward traces with informative assistant text. False - * negatives (an informative trace mistakenly labelled chatter) still - * get rescued because they sit in the second half of the ordered list. - */ -function isInformativeCandidate(r: RankedCandidate): boolean { - const c = r.candidate; - if (c.refKind !== "trace") return true; - const t = c as TraceCandidate; - if ((t.summary?.trim().length ?? 0) > 0) return true; - if ((t.reflection?.trim().length ?? 0) > 0) return true; - const agent = t.agentText?.trim() ?? ""; - if (agent.length === 0) return false; - if (isShortAck(agent)) return false; - return agent.length >= INFORMATIVE_AGENT_TEXT_MIN_CHARS; -} - -function isShortAck(text: string): boolean { - return SHORT_ACK_PATTERNS.some((re) => re.test(text)); -} - /** * Mechanical fail-closed: when the LLM is unavailable / errored, * apply a relative-relevance cutoff so we don't dump the entire ranked diff --git a/apps/memos-local-plugin/core/retrieval/profile.ts b/apps/memos-local-plugin/core/retrieval/profile.ts new file mode 100644 index 000000000..0eb4c767b --- /dev/null +++ b/apps/memos-local-plugin/core/retrieval/profile.ts @@ -0,0 +1,28 @@ +import type { RetrievalProfile } from "./types.js"; + +const PERSONAL_FACT_MARKERS = + /(?:喜欢|最爱|爱吃|爱喝|爱听|偏好|讨厌|不喜欢|名字|姓名|生日|家乡|住在|职业|工作|宠物|歌手|水果|食物|颜色|称呼)/u; +const FIRST_PERSON_MARKERS = /(?:我|我的|本人|关于我)/u; +const PROCEDURAL_MARKERS = + /(?:怎么|如何|怎样).*(?:修|解决|处理|配置|安装|实现|排查)|(?:修复|解决|排查).*(?:问题|故障|报错)/u; +const ENGLISH_PERSONAL_FACT = + /\b(?:what|who|where|when).*\b(?:my|i)\b.*\b(?:favorite|like|love|prefer|name|birthday|live|job|pet)\b|\b(?:my favorite|what do i like|what i like)\b/i; + +/** + * Conservative, deterministic routing for query-specific retrieval policy. + * + * Only high-confidence first-person facts are specialised. Everything else + * stays on the legacy/default route so troubleshooting and task recall do not + * accidentally lose Tier-1 skills. + */ +export function classifyRetrievalProfile(userText: string): RetrievalProfile { + const text = userText.trim(); + if (!text || PROCEDURAL_MARKERS.test(text)) return "default"; + if ( + (FIRST_PERSON_MARKERS.test(text) && PERSONAL_FACT_MARKERS.test(text)) || + ENGLISH_PERSONAL_FACT.test(text) + ) { + return "personal_fact"; + } + return "default"; +} diff --git a/apps/memos-local-plugin/core/retrieval/query-builder.ts b/apps/memos-local-plugin/core/retrieval/query-builder.ts index 940edae15..a64d0db05 100644 --- a/apps/memos-local-plugin/core/retrieval/query-builder.ts +++ b/apps/memos-local-plugin/core/retrieval/query-builder.ts @@ -11,6 +11,7 @@ import { extractErrorSignatures } from "../capture/error-signature.js"; import { + extractExactIdentifiers, extractPatternTerms, prepareFtsMatch, type FtsTokenizerMode, @@ -68,6 +69,12 @@ export interface CompiledQuery { * `LIKE %term%` clause in `searchByPattern`. Empty array = skip. */ patternTerms: string[]; + /** + * Concrete long identifiers that require exact candidate confirmation. + * Kept separate from pattern terms so generic CJK bigrams cannot satisfy + * the precision guard. + */ + exactIdentifiers: string[]; /** Did we truncate the text? Useful for logs. */ truncated: boolean; } @@ -79,10 +86,11 @@ export interface CompiledQuery { export function buildQuery(ctx: RetrievalCtx, opts: BuildQueryOptions = {}): CompiledQuery { switch (ctx.reason) { case "turn_start": { - const hintText = hintToText(ctx.contextHints); - const parts = [ctx.userText?.trim() ?? ""]; - if (hintText) parts.push(hintText); - return finalize(parts.join("\n"), opts); + // contextHints carry host routing / transport metadata (workspace, + // channel, message id, sender id, etc.). They are useful to the + // runtime, but pollute both embeddings and lexical channels when + // mixed into the user's semantic query. + return finalize(ctx.userText?.trim() ?? "", opts); } case "tool_driven": { if (typeof ctx.args?.query === "string" && ctx.args.query.trim()) { @@ -117,6 +125,7 @@ export function buildQuery(ctx: RetrievalCtx, opts: BuildQueryOptions = {}): Com structuralFragments: [], ftsMatch: null, patternTerms: [], + exactIdentifiers: [], truncated: false, }; } @@ -143,6 +152,7 @@ function finalize(raw: string, opts: BuildQueryOptions): CompiledQuery { structuralFragments: [], ftsMatch: null, patternTerms: [], + exactIdentifiers: [], truncated: false, }; } @@ -159,6 +169,7 @@ function finalize(raw: string, opts: BuildQueryOptions): CompiledQuery { // helpers themselves. const ftsMatch = prepareFtsMatch(trimmed, { tokenizer: opts.ftsTokenizer }); const patternTerms = extractPatternTerms(trimmed); + const exactIdentifiers = extractExactIdentifiers(trimmed); if (trimmed.length <= MAX_QUERY_CHARS) { return { text: trimmed, @@ -166,6 +177,7 @@ function finalize(raw: string, opts: BuildQueryOptions): CompiledQuery { structuralFragments, ftsMatch, patternTerms, + exactIdentifiers, truncated: false, }; } @@ -178,6 +190,7 @@ function finalize(raw: string, opts: BuildQueryOptions): CompiledQuery { structuralFragments, ftsMatch, patternTerms, + exactIdentifiers, truncated: true, }; } @@ -190,22 +203,3 @@ function renderArgs(args: Record | undefined): string { return String(args); } } - -function hintToText(hints: Record | undefined): string { - if (!hints) return ""; - const entries = Object.entries(hints).slice(0, 8); - if (entries.length === 0) return ""; - const lines = entries.map(([k, v]) => `${k}: ${renderHintValue(v)}`); - return lines.join("\n"); -} - -function renderHintValue(v: unknown): string { - if (v === null || v === undefined) return ""; - if (typeof v === "string") return v; - if (typeof v === "number" || typeof v === "boolean") return String(v); - try { - return JSON.stringify(v); - } catch { - return String(v); - } -} diff --git a/apps/memos-local-plugin/core/retrieval/ranker.ts b/apps/memos-local-plugin/core/retrieval/ranker.ts index 8c7c1559c..b707c045a 100644 --- a/apps/memos-local-plugin/core/retrieval/ranker.ts +++ b/apps/memos-local-plugin/core/retrieval/ranker.ts @@ -18,13 +18,20 @@ * base-score band — not a dominant term that washes out the RRF * signal. * - * 4. **Multi-channel bypass.** Any candidate surfaced by ≥ 2 channels - * is exempt from the relative-threshold drop (it can still lose in - * MMR on redundancy). This is the backstop that guarantees a - * keyword-only hit confirmed by vector can never be silently - * dropped because a noisy topRelevance dragged the floor up. + * 4. **Hard eligibility before destructive selection.** Long-identifier + * queries reject ordinary vector-only memories before dedupe/MMR while + * preserving the existing skill/world-model exemption. * - * 5. **Smart-seed MMR.** Phase A seeds at most one candidate per tier, + * 5. **Two-stage repeated-question hard dedupe.** Before the threshold, + * exact duplicates collapse only within the same bypass-capability + * partition. After eligibility is resolved, surviving variants collapse + * again so duplicate text still consumes one Top-K slot. + * + * 6. **Strong multi-channel bypass.** Candidates surfaced through at least + * two channels, with one strong primary signal, may bypass the relative + * threshold but still participate normally in MMR. + * + * 7. **Smart-seed MMR.** Phase A seeds at most one candidate per tier, * and only if its relevance is within `smartSeedRatio` of the pool * top. Prevents "force-inject an irrelevant Tier-1 / Tier-3 just * because the tier had a candidate". @@ -33,14 +40,18 @@ */ import { cosinePrenormed, norm2 } from "../storage/vector.js"; +import type { InjectionScoreDetails } from "../../agent-contract/dto.js"; import type { EmbeddingVector } from "../types.js"; import { priorityFor } from "../reward/backprop.js"; +import { normalizeIdentifierText } from "../storage/keyword.js"; +import { dedupeRepeatedQuestionCandidates } from "./candidate-dedupe.js"; import type { ChannelRank, EpisodeCandidate, ExperienceCandidate, RetrievalChannel, RetrievalConfig, + RetrievalProfile, SkillCandidate, TierCandidate, TierKind, @@ -58,6 +69,9 @@ export interface RankerInput { limit: number; config: RetrievalConfig; now: number; + profile?: RetrievalProfile; + /** Concrete long identifiers requiring exact confirmation. */ + exactIdentifiers?: readonly string[]; } export interface RankedCandidate { @@ -77,6 +91,8 @@ export interface RankedCandidate { /** True when this candidate was allowed past the threshold via the * multi-channel bypass (useful for logs / "why did this survive?"). */ bypassedThreshold?: boolean; + /** Structured explanation copied into logs with the final snippet. */ + scoreDetails?: InjectionScoreDetails; } export interface RankerResult { @@ -89,6 +105,12 @@ export interface RankerResult { topRelevance: number; /** Number of candidates the relative-threshold cut. */ droppedByThreshold: number; + /** Repeated-question trace candidates removed before MMR. */ + dedupedBeforeMmr: number; + /** Subset of `dedupedBeforeMmr` removed after threshold eligibility. */ + dedupedAfterThreshold: number; + /** Candidates rejected by long-identifier keyword confirmation. */ + droppedByKeywordConfirmation: number; /** Absolute floor applied (`topRelevance · floor`). */ thresholdFloor: number; /** Channel hit counts aggregated across all candidates. */ @@ -141,16 +163,81 @@ export function rank(input: RankerInput): RankerResult { kept, topRelevance: 0, droppedByThreshold: 0, + dedupedBeforeMmr: 0, + dedupedAfterThreshold: 0, + droppedByKeywordConfirmation: 0, thresholdFloor: 0, channelHits, }; } assignChannelRrf(bag, input.config.rrfConstant); - for (const c of bag) c.relevance += RRF_WEIGHT * c.rrf; + for (const c of bag) { + const semantic = bestChannelScore(c.candidate); + const tierBoost = c.relevance - semantic; + const rrfBoost = RRF_WEIGHT * c.rrf; + c.relevance += rrfBoost; + c.scoreDetails = { + profile: input.profile ?? "default", + semantic, + tierBoost, + rrfBoost, + relevance: c.relevance, + mmrLambda: 0, + redundancy: 0, + finalScore: c.relevance, + channels: (c.candidate.channels ?? []).map((channel) => channel.channel), + bypassedThreshold: false, + }; + } - // ─── 2. Relative threshold cut (with multi-channel bypass) ──────────────── - const topRelevance = bag.reduce((m, c) => Math.max(m, c.relevance), 0); + // ─── 2. Query-specific hard eligibility ─────────────────────────────────── + // Long identifiers are unsafe to match through embedding similarity or + // generic short-pattern hits alone. Apply exact confirmation before any + // irreversible dedupe/MMR choice. + const exactIdentifiers = input.exactIdentifiers ?? []; + const keywordEligible = exactIdentifiers.length > 0 + ? bag.filter((candidate) => + isIdentifierEligible(candidate.candidate, exactIdentifiers) + ) + : bag; + const droppedByKeywordConfirmation = bag.length - keywordEligible.length; + + if (keywordEligible.length === 0) { + return { + ranked: [], + tierSizes, + kept, + topRelevance: 0, + droppedByThreshold: 0, + dedupedBeforeMmr: 0, + dedupedAfterThreshold: 0, + droppedByKeywordConfirmation, + thresholdFloor: 0, + channelHits, + }; + } + + // ─── 3. High-confidence hard dedupe ─────────────────────────────────────── + // Relevance and RRF are already available, so exact duplicate evidence keeps + // its strongest representative. Dedupe happens before deriving the relative + // threshold so repeated high-scoring chatter cannot inflate the cutoff. + // + // Bypass-capable and ordinary variants remain in separate partitions until + // the threshold has been evaluated. Otherwise a slightly higher ordinary + // duplicate could erase the only variant entitled to bypass the floor. + const deduped = dedupeRepeatedQuestionCandidates(keywordEligible, { + capabilityKey: (candidate) => + canBypassRelativeThreshold(candidate.candidate, input.config) + ? "threshold-bypass" + : "normal", + }); + + // ─── 4. Relative threshold cut (with multi-channel bypass) ──────────────── + const topRelevance = deduped.kept.reduce( + (maximum, candidate) => Math.max(maximum, candidate.relevance), + 0, + ); const floorRatio = input.config.relativeThresholdFloor ?? DEFAULT_RELATIVE_THRESHOLD; const cutoff = topRelevance > 0 ? topRelevance * floorRatio : 0; @@ -158,10 +245,12 @@ export function rank(input: RankerInput): RankerResult { let droppedByThreshold = 0; const survivors: RankedCandidate[] = []; - for (const c of bag) { - const channels = c.candidate.channels ?? []; - const multiChannel = bypassEnabled && channels.length >= 2; + for (const c of deduped.kept) { + const multiChannel = + bypassEnabled && + canBypassRelativeThreshold(c.candidate, input.config); if (multiChannel) c.bypassedThreshold = true; + if (c.scoreDetails) c.scoreDetails.bypassedThreshold = multiChannel; if (cutoff > 0 && c.relevance < cutoff && !multiChannel) { droppedByThreshold += 1; continue; @@ -176,18 +265,31 @@ export function rank(input: RankerInput): RankerResult { kept, topRelevance, droppedByThreshold, + dedupedBeforeMmr: deduped.dropped.length, + dedupedAfterThreshold: 0, + droppedByKeywordConfirmation, thresholdFloor: cutoff, channelHits, }; } - // ─── 3. MMR-style greedy pick ───────────────────────────────────────────── - const λ = clamp(input.config.mmrLambda, 0, 1); + // ─── 5. Collapse capability variants after threshold eligibility ────────── + // Once every survivor has cleared the normal floor or its bypass, capability + // variants are equivalent again. Collapse them before MMR so duplicate text + // still consumes only one Top-K slot. + const postThresholdDeduped = + dedupeRepeatedQuestionCandidates(survivors); + + // ─── 6. MMR-style greedy pick ───────────────────────────────────────────── + const λ = + input.profile === "personal_fact" + ? 0.85 + : clamp(input.config.mmrLambda, 0, 1); const out: RankedCandidate[] = []; const selectedVecs: EmbeddingVector[] = []; const selectedNorms: number[] = []; - const pool = [...survivors]; - const limit = Math.min(input.limit, survivors.length); + const pool = [...postThresholdDeduped.kept]; + const limit = Math.min(input.limit, postThresholdDeduped.kept.length); const smartSeed = input.config.smartSeed !== false; const seedRatio = smartSeed ? input.config.smartSeedRatio ?? DEFAULT_SMART_SEED_RATIO @@ -222,6 +324,7 @@ export function rank(input: RankerInput): RankerResult { if (tierBestRel < seedCutoff) continue; const c = pool.splice(bestIdx, 1)[0]!; c.score = bestScore; + finaliseScoreDetails(c, selectedVecs, selectedNorms, λ); out.push(c); kept[tk] += 1; pushVec(selectedVecs, selectedNorms, c); @@ -242,6 +345,7 @@ export function rank(input: RankerInput): RankerResult { if (bestIdx < 0) break; const [picked] = pool.splice(bestIdx, 1); picked!.score = bestScore; + finaliseScoreDetails(picked!, selectedVecs, selectedNorms, λ); out.push(picked!); kept[picked!.candidate.tier] += 1; pushVec(selectedVecs, selectedNorms, picked!); @@ -256,11 +360,27 @@ export function rank(input: RankerInput): RankerResult { kept, topRelevance, droppedByThreshold, + dedupedBeforeMmr: + deduped.dropped.length + postThresholdDeduped.dropped.length, + dedupedAfterThreshold: postThresholdDeduped.dropped.length, + droppedByKeywordConfirmation, thresholdFloor: cutoff, channelHits, }; } +function finaliseScoreDetails( + candidate: RankedCandidate, + selected: readonly EmbeddingVector[], + selectedNorms: readonly number[], + lambda: number, +): void { + if (!candidate.scoreDetails) return; + candidate.scoreDetails.mmrLambda = lambda; + candidate.scoreDetails.redundancy = maxCos(candidate, selected, selectedNorms); + candidate.scoreDetails.finalScore = candidate.score; +} + // ─── Helpers ──────────────────────────────────────────────────────────────── /** @@ -294,7 +414,7 @@ function relevanceFor(c: TierCandidate, input: RankerInput): number { input.config.decayHalfLifeDays, input.now, ); - const blend = priorityBlendFor(input.config); + const blend = priorityBlendFor(input.config, input.profile); return base + blend * live; } if (c.refKind === "episode") { @@ -305,7 +425,7 @@ function relevanceFor(c: TierCandidate, input: RankerInput): number { input.config.decayHalfLifeDays, input.now, ); - const blend = priorityBlendFor(input.config); + const blend = priorityBlendFor(input.config, input.profile); return base + blend * live; } if (c.refKind === "experience") { @@ -326,7 +446,11 @@ function relevanceFor(c: TierCandidate, input: RankerInput): number { * 0.3). Configs that explicitly set `weightPriority` higher than that * still work — their intent "priority matters more" is preserved. */ -function priorityBlendFor(config: RetrievalConfig): number { +function priorityBlendFor( + config: RetrievalConfig, + profile: RetrievalProfile | undefined, +): number { + if (profile === "personal_fact") return 0; const w = config.weightPriority; if (w == null || w <= 0) return 0; // Cap the effective blend so priority can't single-handedly push a @@ -335,6 +459,71 @@ function priorityBlendFor(config: RetrievalConfig): number { return Math.min(w, DEFAULT_PRIORITY_BLEND); } +function canBypassRelativeThreshold( + candidate: TierCandidate, + config: RetrievalConfig, +): boolean { + const channels = candidate.channels ?? []; + if (config.multiChannelBypass === false || channels.length < 2) return false; + return (candidate.channels ?? []).some((channel) => { + if ( + channel.channel === "vec" || + channel.channel === "vec_summary" || + channel.channel === "vec_action" + ) { + return channel.score >= config.minTraceSim; + } + if (channel.channel === "structural") return channel.score >= 0.8; + return channel.score >= 0.75; + }); +} + +function isIdentifierEligible( + candidate: TierCandidate, + exactIdentifiers: readonly string[], +): boolean { + if ( + candidate.refKind === "skill" || + candidate.refKind === "world-model" + ) { + return true; + } + if ( + (candidate.channels ?? []).some( + (channel) => channel.channel === "exact_identifier", + ) + ) { + return true; + } + const body = normalizeIdentifierText(identifierSearchText(candidate)); + return exactIdentifiers.some((identifier) => + body.includes(normalizeIdentifierText(identifier)) + ); +} + +function identifierSearchText(candidate: TierCandidate): string { + if (candidate.refKind === "trace") { + return [ + candidate.userText, + candidate.agentText, + candidate.summary, + candidate.reflection, + ...candidate.tags, + ].filter(Boolean).join("\n"); + } + if (candidate.refKind === "episode") return candidate.summary; + if (candidate.refKind === "experience") { + return [ + candidate.title, + candidate.trigger, + candidate.procedure, + candidate.verification, + candidate.boundary, + ].filter(Boolean).join("\n"); + } + return ""; +} + function bestChannelScore(c: TierCandidate): number { const channels = c.channels ?? []; if (channels.length === 0) { @@ -392,9 +581,17 @@ function maxCos( if (candNorm === 0) return 0; let m = 0; for (let i = 0; i < selected.length; i += 1) { + const selectedVec = selected[i]!; + // Embedding migrations can leave the candidate pool temporarily + // heterogeneous (for example, legacy 384-dim rows mixed with current + // 1024-dim rows recalled through FTS/pattern channels). Those vectors + // do not share a coordinate space, so their redundancy is undefined. + // Keep both candidates and skip only this pairwise comparison instead + // of aborting the entire retrieval packet. + if (vec.length !== selectedVec.length) continue; const sn = Math.sqrt(selectedNorms[i]!); if (sn === 0) continue; - const sim = cosinePrenormed(vec, candNorm, selected[i]!, selectedNorms[i]!); + const sim = cosinePrenormed(vec, candNorm, selectedVec, selectedNorms[i]!); if (sim > m) m = sim; } return m; diff --git a/apps/memos-local-plugin/core/retrieval/retrieve.ts b/apps/memos-local-plugin/core/retrieval/retrieve.ts index da4604589..11a998349 100644 --- a/apps/memos-local-plugin/core/retrieval/retrieve.ts +++ b/apps/memos-local-plugin/core/retrieval/retrieve.ts @@ -34,7 +34,7 @@ import type { RetrievalEventBus } from "./events.js"; import { dedupeTraceEpisodeByEpisodeId } from "./dedupe-trace-episode.js"; import { toPacket, renderSnippetForDebug } from "./injector.js"; import { llmFilterCandidates } from "./llm-filter.js"; -import { rank, type RankedCandidate } from "./ranker.js"; +import { rank } from "./ranker.js"; import { runTier1 } from "./tier1-skill.js"; import { runTier2Experience } from "./tier2-experience.js"; import { runTier2 } from "./tier2-trace.js"; @@ -45,6 +45,7 @@ import type { RetrievalCtx, RetrievalDeps, RetrievalResult, + RetrievalProfile, RetrievalStats, SkillCandidate, TraceCandidate, @@ -78,6 +79,7 @@ export interface RetrieveOptions { export interface RetrievePlanOverride { scenarioId?: string; + profile?: RetrievalProfile; wantTier1?: boolean; wantTier2?: boolean; wantTier3?: boolean; @@ -205,6 +207,7 @@ export async function repairRetrieve( interface RunPlan { scenarioId?: string; + profile?: RetrievalProfile; wantTier1: boolean; wantTier2: boolean; wantTier3: boolean; @@ -218,6 +221,7 @@ function applyPlanOverride(plan: RunPlan, override?: RetrievePlanOverride): RunP return { ...plan, scenarioId: override.scenarioId ?? plan.scenarioId, + profile: override.profile ?? plan.profile, wantTier1: override.wantTier1 ?? plan.wantTier1, wantTier2: override.wantTier2 ?? plan.wantTier2, wantTier3: override.wantTier3 ?? plan.wantTier3, @@ -276,14 +280,20 @@ async function runAll( // so we no longer short-circuit on `emptyVec`. We only require *some* // channel to be armed. const haveKeywordChannel = - !!compiled.ftsMatch || (compiled.patternTerms?.length ?? 0) > 0; + !!compiled.ftsMatch || + compiled.patternTerms.length > 0 || + compiled.exactIdentifiers.length > 0; const noUsableChannel = !queryVec && !haveKeywordChannel; // Kick off the tiers in parallel — each resolves to its own list. const wantTier1 = plan.wantTier1 && deps.config.tier1TopK > 0; const wantTier2 = plan.wantTier2 && deps.config.tier2TopK > 0; const wantTier3 = plan.wantTier3 && deps.config.tier3TopK > 0; - const traceOnly = plan.traceOnly === true || deps.config.lightweightMemory === true; + const profile = plan.profile ?? "default"; + const traceOnly = + plan.traceOnly === true || + deps.config.lightweightMemory === true || + profile === "personal_fact"; const tier1Start = Date.now(); const tier1Promise: Promise = @@ -296,6 +306,7 @@ async function runAll( rawText: compiled.text, ftsMatch: compiled.ftsMatch, patternTerms: compiled.patternTerms, + exactIdentifiers: compiled.exactIdentifiers, }, ) : Promise.resolve([]); @@ -311,6 +322,8 @@ async function runAll( structuralFragments: compiled.structuralFragments, ftsMatch: compiled.ftsMatch, patternTerms: compiled.patternTerms, + exactIdentifiers: compiled.exactIdentifiers, + profile, includeLowValue: plan.includeLowValue, excludeSessionId: ctx.reason === "turn_start" && sessionId && !deps.config.lightweightMemory @@ -328,6 +341,7 @@ async function runAll( queryVec, ftsMatch: compiled.ftsMatch, patternTerms: compiled.patternTerms, + exactIdentifiers: compiled.exactIdentifiers, }, ) : Promise.resolve([]); @@ -341,6 +355,7 @@ async function runAll( queryVec: queryVec ?? null, ftsMatch: compiled.ftsMatch, patternTerms: compiled.patternTerms, + exactIdentifiers: compiled.exactIdentifiers, }, ) : Promise.resolve([]); @@ -353,33 +368,33 @@ async function runAll( ]); const tier1LatencyMs = wantTier1 ? Date.now() - tier1Start : 0; - const tier2LatencyMs = wantTier2 ? Date.now() - tier2Start : 0; + let tier2LatencyMs = wantTier2 ? Date.now() - tier2Start : 0; const tier3LatencyMs = wantTier3 ? Date.now() - tier3Start : 0; const fuseStart = Date.now(); - const rawCandidateCount = + let activeTier2 = tier2; + let rawCandidateCount = tier1.length + - tier2.traces.length + - tier2.episodes.length + + activeTier2.traces.length + + activeTier2.episodes.length + tier2Experiences.length + tier3.length; - const ranked = rank({ + const exactIdentifiers = + ctx.reason === "decision_repair" ? [] : compiled.exactIdentifiers; + let ranked = rank({ tier1, - tier2Traces: tier2.traces, - tier2Episodes: traceOnly ? [] : tier2.episodes, + tier2Traces: activeTier2.traces, + tier2Episodes: traceOnly ? [] : activeTier2.episodes, tier2Experiences, tier3, limit: plan.limit, config: deps.config, now: deps.now(), + profile, + exactIdentifiers, }); - const mechanicalRanked = ctx.reason !== "decision_repair" && - requiresKeywordConfirmation(compiled.text) - ? ranked.ranked.filter((candidate) => - bypassesKeywordConfirmation(candidate) || hasKeywordChannel(candidate) - ) - : ranked.ranked; - const fuseLatencyMs = Date.now() - fuseStart; + let mechanicalRanked = ranked.ranked; + let fuseLatencyMs = Date.now() - fuseStart; // ─── LLM relevance filter ────────────────────────────────────────── // Mechanical retrieval produces high-recall but low-precision @@ -389,7 +404,7 @@ async function runAll( // keeps the mechanical ranking so local lightweight memories remain // searchable in offline/default installs. const queryText = (ctx as { userText?: string }).userText ?? compiled.text ?? ""; - const filterResult = opts.skipLlmFilter + let filterResult = opts.skipLlmFilter ? { kept: mechanicalRanked, dropped: [], @@ -397,14 +412,92 @@ async function runAll( sufficient: null, } : await llmFilterCandidates( - { query: queryText, ranked: mechanicalRanked, episodeId }, + { query: queryText, ranked: mechanicalRanked, episodeId, profile }, { llm: deps.llm ?? null, log, config: deps.config, }, ); + + let retrievalPasses = 1; + if ( + profile === "personal_fact" && + filterResult.kept.length === 0 && + (filterResult.outcome === "llm_filtered_empty" || + mechanicalRanked.length === 0) && + wantTier2 && + queryVec && + !opts.skipLlmFilter + ) { + const wideningStartedAt = Date.now(); + const widenedTier2 = await runTier2( + { repos: deps.repos, config: deps.config, now: deps.now }, + { + queryVec, + tags: compiled.tags, + structuralFragments: compiled.structuralFragments, + ftsMatch: compiled.ftsMatch, + patternTerms: compiled.patternTerms, + exactIdentifiers: compiled.exactIdentifiers, + profile, + includeActionVector: true, + includeLowValue: plan.includeLowValue, + }, + ); + tier2LatencyMs += Date.now() - wideningStartedAt; + + const wideningFuseStartedAt = Date.now(); + const widenedRanked = rank({ + tier1: [], + tier2Traces: widenedTier2.traces, + tier2Episodes: [], + tier2Experiences: [], + tier3: [], + limit: plan.limit, + config: deps.config, + now: deps.now(), + profile, + exactIdentifiers, + }); + const widenedMechanical = widenedRanked.ranked; + fuseLatencyMs += Date.now() - wideningFuseStartedAt; + const widenedFilter = await llmFilterCandidates( + { + query: queryText, + ranked: widenedMechanical, + episodeId, + profile, + }, + { + llm: deps.llm ?? null, + log, + config: deps.config, + }, + ); + + activeTier2 = widenedTier2; + ranked = widenedRanked; + mechanicalRanked = widenedMechanical; + filterResult = widenedFilter; + rawCandidateCount = widenedTier2.traces.length; + retrievalPasses = 2; + log.debug("retrieval.widened", { + profile, + pass: retrievalPasses, + actionChannel: true, + candidates: rawCandidateCount, + outcome: widenedFilter.outcome, + }); + } const filtered = filterResult; + for (const candidate of mechanicalRanked) { + log.debug("rank.candidate", { + refKind: candidate.candidate.refKind, + refId: candidate.candidate.refId, + ...candidate.scoreDetails, + }); + } log.debug("llm_filter.done", { outcome: filtered.outcome, enforced: false, @@ -412,6 +505,9 @@ async function runAll( raw: rawCandidateCount, afterThreshold: mechanicalRanked.length, droppedByThreshold: ranked.droppedByThreshold, + dedupedBeforeMmr: ranked.dedupedBeforeMmr, + dedupedAfterThreshold: ranked.dedupedAfterThreshold, + droppedByKeywordConfirmation: ranked.droppedByKeywordConfirmation, thresholdFloor: round(ranked.thresholdFloor, 3), topRelevance: round(ranked.topRelevance, 3), kept: filtered.kept.length, @@ -471,12 +567,21 @@ async function runAll( // "initial N → kept M" without the viewer having to re-run the // mechanical pipeline. packet.droppedByLlm = filtered.dropped - .map((r) => renderSnippetForDebug(r.candidate)) + .map((r) => { + const snippet = renderSnippetForDebug(r.candidate); + if (snippet) { + snippet.score = round(r.score, 4); + snippet.scoreDetails = r.scoreDetails; + } + return snippet; + }) .filter((s): s is NonNullable => s !== null); const stats: RetrievalStats = { reason: ctx.reason, scenarioId: plan.scenarioId, + profile, + retrievalPasses, agent, sessionId, episodeId, @@ -486,7 +591,10 @@ async function runAll( tier3: plan.wantTier3, }, tier1Count: tier1.length, - tier2Count: tier2.traces.length + (traceOnly ? 0 : tier2.episodes.length) + tier2Experiences.length, + tier2Count: + activeTier2.traces.length + + (traceOnly ? 0 : activeTier2.episodes.length) + + tier2Experiences.length, tier3Count: tier3.length, tier1LatencyMs, tier2LatencyMs, @@ -495,10 +603,15 @@ async function runAll( totalLatencyMs: Date.now() - ts, queryTokens: approxTokens(compiled.text), queryTags: compiled.tags, + exactIdentifierCount: compiled.exactIdentifiers.length, emptyPacket: packet.snippets.length === 0, embedding: embeddingStats, rawCandidateCount, droppedByThresholdCount: ranked.droppedByThreshold, + dedupedBeforeMmrCount: ranked.dedupedBeforeMmr, + dedupedAfterThresholdCount: ranked.dedupedAfterThreshold, + droppedByKeywordConfirmationCount: + ranked.droppedByKeywordConfirmation, thresholdFloor: ranked.thresholdFloor, topRelevance: ranked.topRelevance, rankedCount: mechanicalRanked.length, @@ -515,8 +628,8 @@ async function runAll( reason: ctx.reason, sessionId, tier1: tier1.length, - tier2: tier2.traces.length, - tier2Ep: traceOnly ? 0 : tier2.episodes.length, + tier2: activeTier2.traces.length, + tier2Ep: traceOnly ? 0 : activeTier2.episodes.length, tier2Experience: tier2Experiences.length, tier3: tier3.length, kept: packet.snippets.length, @@ -596,28 +709,6 @@ function emptyResult( }; } -function requiresKeywordConfirmation(text: string): boolean { - const tokens = text.match(/[A-Za-z0-9_:-]{12,}/g) ?? []; - return tokens.some((token) => { - const hasIdentifierShape = /[_:-]/.test(token) || /\d/.test(token); - const hasEnoughEntropy = /[A-Za-z]/.test(token) && token.length >= 16; - return hasIdentifierShape && hasEnoughEntropy; - }); -} - -function hasKeywordChannel(candidate: RankedCandidate): boolean { - return (candidate.candidate.channels ?? []).some((channel) => - channel.channel === "fts" || - channel.channel === "pattern" || - channel.channel === "structural" - ); -} - -function bypassesKeywordConfirmation(candidate: RankedCandidate): boolean { - const refKind = candidate.candidate.refKind; - return refKind === "skill" || refKind === "world-model"; -} - function approxTokens(s: string): number { if (!s) return 0; return Math.ceil(s.length / 4); diff --git a/apps/memos-local-plugin/core/retrieval/tier1-skill.ts b/apps/memos-local-plugin/core/retrieval/tier1-skill.ts index ee621440b..bc0de1354 100644 --- a/apps/memos-local-plugin/core/retrieval/tier1-skill.ts +++ b/apps/memos-local-plugin/core/retrieval/tier1-skill.ts @@ -50,19 +50,20 @@ export type Tier1Input = rawText: string; ftsMatch?: string | null; patternTerms?: readonly string[]; + exactIdentifiers?: readonly string[]; } | { kind: "raw"; text: string; ftsMatch?: string | null; patternTerms?: readonly string[]; + exactIdentifiers?: readonly string[]; }; interface CandidateState { cosine: number; channels: ChannelRank[]; meta: { name: string; status: SkillStatus; eta: number; gain: number }; - vec: EmbeddingVector | null; } export async function runTier1( @@ -75,11 +76,15 @@ export async function runTier1( const queryVec = await resolveVec(deps, input); const ftsMatch = "ftsMatch" in input ? input.ftsMatch ?? null : null; const patternTerms = "patternTerms" in input ? input.patternTerms ?? [] : []; + const exactIdentifiers = + "exactIdentifiers" in input ? input.exactIdentifiers ?? [] : []; const haveVec = !!queryVec && queryVec.length > 0; const haveFts = !!ftsMatch && !!repos.skills.searchByText; const havePattern = patternTerms.length > 0 && !!repos.skills.searchByPattern; - if (!haveVec && !haveFts && !havePattern) { + const haveExact = + exactIdentifiers.length > 0 && !!repos.skills.searchByPattern; + if (!haveVec && !haveFts && !havePattern && !haveExact) { log.debug("empty_query", { reason: "no channels armed" }); return []; } @@ -106,7 +111,6 @@ export async function runTier1( rank: idx, score: h.score, meta: h.meta, - vec: queryVec!, }); }); } @@ -120,7 +124,6 @@ export async function runTier1( rank: idx, score: h.score, meta: h.meta, - vec: queryVec ?? null, }); }); } @@ -136,7 +139,23 @@ export async function runTier1( rank: idx, score: h.score, meta: h.meta, - vec: queryVec ?? null, + }); + }); + } + + if (haveExact) { + const exactHits = repos.skills.searchByPattern!( + exactIdentifiers, + keywordPoolSize, + { statusIn }, + ); + exactHits.forEach((h, idx) => { + upsertCandidate(merged, h.id as SkillId, { + cosine: 0, + channel: "exact_identifier", + rank: idx, + score: 1, + meta: h.meta, }); }); } @@ -165,7 +184,7 @@ export async function runTier1( refId: sk.id, cosine: state.cosine, ts: Date.now(), - vec: state.vec, + vec: sk.vec ?? null, skillName: sk.name, eta: sk.eta, status: sk.status, @@ -189,6 +208,7 @@ export async function runTier1( vec: haveVec, fts: haveFts, pattern: havePattern, + exactIdentifier: haveExact, }, latencyMs: Date.now() - startedAt, }); @@ -245,7 +265,6 @@ function upsertCandidate( rank: number; score: number; meta?: { name: string; status: SkillStatus; eta: number; gain: number }; - vec: EmbeddingVector | null; }, ): void { const entry = into.get(id); @@ -255,13 +274,11 @@ function upsertCandidate( cosine: patch.cosine, channels: [{ channel: patch.channel, rank: patch.rank, score: patch.score }], meta: patch.meta, - vec: patch.vec, }); return; } entry.channels.push({ channel: patch.channel, rank: patch.rank, score: patch.score }); if (patch.cosine > entry.cosine) entry.cosine = patch.cosine; - if (!entry.vec && patch.vec) entry.vec = patch.vec; } function bestChannelScore(c: SkillCandidate): number { diff --git a/apps/memos-local-plugin/core/retrieval/tier2-experience.ts b/apps/memos-local-plugin/core/retrieval/tier2-experience.ts index f6103de44..55dca0ba6 100644 --- a/apps/memos-local-plugin/core/retrieval/tier2-experience.ts +++ b/apps/memos-local-plugin/core/retrieval/tier2-experience.ts @@ -20,6 +20,7 @@ export interface Tier2ExperienceInput { queryVec: EmbeddingVector | null; ftsMatch?: string | null; patternTerms?: string[]; + exactIdentifiers?: string[]; } export async function runTier2Experience( @@ -46,7 +47,11 @@ export async function runTier2Experience( const haveFts = !!input.ftsMatch && !!repo.searchByText; const havePattern = !!input.patternTerms && input.patternTerms.length > 0 && !!repo.searchByPattern; - if (!haveVec && !haveFts && !havePattern) { + const haveExact = + !!input.exactIdentifiers && + input.exactIdentifiers.length > 0 && + !!repo.searchByPattern; + if (!haveVec && !haveFts && !havePattern && !haveExact) { return []; } @@ -63,7 +68,6 @@ export async function runTier2Experience( channel: "vec", rank: idx, score: hit.score, - vec: input.queryVec!, }); }); } @@ -76,7 +80,6 @@ export async function runTier2Experience( channel: "fts", rank: idx, score: hit.score, - vec: input.queryVec ?? null, }); }); } @@ -89,7 +92,22 @@ export async function runTier2Experience( channel: "pattern", rank: idx, score: hit.score, - vec: input.queryVec ?? null, + }); + }); + } + + if (haveExact) { + const hits = repo.searchByPattern!( + input.exactIdentifiers!, + keywordPoolSize, + { statusIn }, + ); + hits.forEach((hit, idx) => { + upsertCandidate(merged, hit.id as PolicyId, { + cosine: 0, + channel: "exact_identifier", + rank: idx, + score: 1, }); }); } @@ -107,7 +125,7 @@ export async function runTier2Experience( refId: row.id as PolicyId, cosine: state.cosine, ts: row.updatedAt ?? Date.now(), - vec: row.vec ?? state.vec, + vec: row.vec ?? null, channels: state.channels, title: row.title, trigger: row.trigger ?? "", @@ -143,6 +161,7 @@ export async function runTier2Experience( vec: haveVec, fts: haveFts, pattern: havePattern, + exactIdentifier: haveExact, }, latencyMs: Date.now() - startedAt, }); @@ -159,7 +178,6 @@ export async function runTier2Experience( interface CandidateState { cosine: number; channels: ChannelRank[]; - vec: EmbeddingVector | null; } function upsertCandidate( @@ -170,7 +188,6 @@ function upsertCandidate( channel: RetrievalChannel; rank: number; score: number; - vec: EmbeddingVector | null; }, ): void { const curr = map.get(id); @@ -183,13 +200,11 @@ function upsertCandidate( map.set(id, { cosine: hit.cosine, channels: [channel], - vec: hit.vec, }); return; } curr.cosine = Math.max(curr.cosine, hit.cosine); curr.channels.push(channel); - if (!curr.vec && hit.vec) curr.vec = hit.vec; } function bestChannelScore(c: ExperienceCandidate): number { diff --git a/apps/memos-local-plugin/core/retrieval/tier2-trace.ts b/apps/memos-local-plugin/core/retrieval/tier2-trace.ts index fad984821..a02a79f2a 100644 --- a/apps/memos-local-plugin/core/retrieval/tier2-trace.ts +++ b/apps/memos-local-plugin/core/retrieval/tier2-trace.ts @@ -25,7 +25,6 @@ */ import { rootLogger } from "../logger/index.js"; -import { priorityFor } from "../reward/backprop.js"; import type { EmbeddingVector, EpisodeId, SessionId, TraceId } from "../types.js"; import type { ChannelRank, @@ -33,6 +32,7 @@ import type { RetrievalChannel, RetrievalConfig, RetrievalEmbedder, + RetrievalProfile, RetrievalRepos, TraceCandidate, TraceVecKind, @@ -64,6 +64,8 @@ export interface Tier2Input { ftsMatch?: string | null; /** Pattern terms (2-char ASCII / CJK bigrams). */ patternTerms?: readonly string[]; + /** Full identifiers searched independently from generic pattern terms. */ + exactIdentifiers?: readonly string[]; /** Whether `decision_repair` forced `includeLowValue`. */ includeLowValue?: boolean; /** @@ -71,6 +73,10 @@ export interface Tier2Input { * turn-start retrieval should not repeat the current chat window). */ excludeSessionId?: SessionId; + /** Query-specific policy; personal facts use summary vectors first. */ + profile?: RetrievalProfile; + /** Controlled second pass may add the action channel for personal facts. */ + includeActionVector?: boolean; } export interface Tier2Result { @@ -98,24 +104,36 @@ export async function runTier2(deps: Tier2Deps, input: Tier2Input): Promise 0) { - const summaryHits = repos.traces.searchByVector(input.queryVec, vecPoolSize, { - kind: "summary", - anyOfTags: tagsForStorage, - where: vectorFilters.where, - params: vectorFilters.params, - hardCap: vecPoolSize * 4, - }); - mergeChannelHits(blended, summaryHits, "vec_summary", input.queryVec); - - if (!config.lightweightMemory) { - const actionHits = repos.traces.searchByVector(input.queryVec, vecPoolSize, { - kind: "action", + const summaryHits = filterVectorHits( + repos.traces.searchByVector(input.queryVec, vecPoolSize, { + kind: "summary", anyOfTags: tagsForStorage, where: vectorFilters.where, params: vectorFilters.params, hardCap: vecPoolSize * 4, - }); - mergeChannelHits(blended, actionHits, "vec_action", input.queryVec); + }), + config, + input.profile, + ); + mergeChannelHits(blended, summaryHits, "vec_summary"); + + if ( + !config.lightweightMemory && + (input.profile !== "personal_fact" || input.includeActionVector === true) + ) { + const actionHits = filterVectorHits( + repos.traces.searchByVector(input.queryVec, vecPoolSize, { + kind: "action", + anyOfTags: tagsForStorage, + where: vectorFilters.where, + params: vectorFilters.params, + hardCap: vecPoolSize * 4, + }), + config, + input.profile, + "action", + ); + mergeChannelHits(blended, actionHits, "vec_action"); } // If both vector channels came back empty AND tag filtering is @@ -124,13 +142,17 @@ export async function runTier2(deps: Tier2Deps, input: Tier2Input): Promise 0 && + repos.traces.searchByPattern + ) { + const exactHits = repos.traces.searchByPattern( + input.exactIdentifiers, + keywordPoolSize, + { + where: searchFilters.where, + params: searchFilters.params, + }, + ); + mergeChannelHits( + blended, + exactHits.map((hit) => ({ ...hit, score: 1 })), + "exact_identifier", + ); } // ─── Structural error-signature channel ─────────────────────────── @@ -193,7 +239,7 @@ export async function runTier2(deps: Tier2Deps, input: Tier2Input): Promise blendScore(b, deps) - blendScore(a, deps)); - const topTraces = traces.slice(0, config.tier2TopK); + // Preserve the expanded pool for the global ranker. Value/priority is + // deliberately absent here; the ranker adds it exactly once. + traces.sort((a, b) => preRankScore(b) - preRankScore(a)); + const candidateCap = Math.max(vecPoolSize, keywordPoolSize); + const pooledTraces = traces.slice(0, candidateCap); // Roll up to episode-level summaries. const episodes = config.lightweightMemory ? [] - : rollupEpisodes(topTraces, deps).slice(0, config.tier2TopK); + : rollupEpisodes(pooledTraces, deps).slice(0, config.tier2TopK); log.info("done", { - traceCount: topTraces.length, + traceCount: pooledTraces.length, episodeCount: episodes.length, keywordPoolSize, vecPoolSize, latencyMs: Date.now() - startedAt, }); - return { traces: topTraces, episodes }; + return { traces: pooledTraces, episodes }; } catch (err) { log.error("failed", { err: { message: err instanceof Error ? err.message : String(err) }, @@ -332,6 +384,24 @@ function resolveTagFilter( return tags; } +function filterVectorHits( + hits: readonly T[], + config: RetrievalConfig, + profile: RetrievalProfile | undefined, + channel: "summary" | "action" = "summary", +): T[] { + if (hits.length === 0) return []; + const topScore = Math.max(...hits.map((hit) => hit.score)); + const adaptiveRatio = + profile === "personal_fact" + ? channel === "action" + ? 0.65 + : 0.55 + : 0.45; + const floor = Math.max(config.minTraceSim, topScore * adaptiveRatio); + return hits.filter((hit) => hit.score >= floor); +} + function mergeChannelHits( into: Map, hits: Array<{ @@ -347,7 +417,6 @@ function mergeChannelHits( }; }>, channel: RetrievalChannel, - queryVec: EmbeddingVector | null, ): void { hits.forEach((h, idx) => { const id = h.id as TraceId; @@ -376,7 +445,9 @@ function mergeChannelHits( // and depend on RRF for ranking. cosine: isVec ? h.score : 0, ts: meta.ts, - vec: queryVec, + // The vector search API returns only ids/scores. Hydration below + // replaces this with the candidate row's actual stored vector. + vec: null, value: meta.value, priority: meta.priority, episodeId: meta.episode_id, @@ -392,10 +463,11 @@ function mergeChannelHits( }); } -function blendScore(c: TraceCandidate, deps: Tier2Deps): number { - // Re-derive priority so we respect any time elapsed between write and - // retrieval (keeps old high-V traces sinking over the half-life). - const livePriority = priorityFor(c.value, c.ts, deps.config.decayHalfLifeDays, deps.now()); +function preRankScore(c: TraceCandidate): number { + const semantic = Math.max( + c.cosine, + ...(c.channels ?? []).map((channel) => channel.score), + ); const channelCount = c.channels?.length ?? 0; const channelLift = channelCount > 1 @@ -403,11 +475,7 @@ function blendScore(c: TraceCandidate, deps: Tier2Deps): number { // a 5-channel match doesn't become 5× anything. Math.min(0.15, 0.04 * (channelCount - 1)) : 0; - return ( - deps.config.weightCosine * c.cosine + - deps.config.weightPriority * livePriority + - channelLift - ); + return semantic + channelLift; } /** diff --git a/apps/memos-local-plugin/core/retrieval/tier3-world.ts b/apps/memos-local-plugin/core/retrieval/tier3-world.ts index 94997ac83..0e5dd56dc 100644 --- a/apps/memos-local-plugin/core/retrieval/tier3-world.ts +++ b/apps/memos-local-plugin/core/retrieval/tier3-world.ts @@ -35,13 +35,13 @@ export interface Tier3Input { queryVec: EmbeddingVector | null; ftsMatch?: string | null; patternTerms?: readonly string[]; + exactIdentifiers?: readonly string[]; } interface CandidateState { cosine: number; channels: ChannelRank[]; meta?: { title: string }; - vec: EmbeddingVector | null; } export async function runTier3( @@ -55,7 +55,11 @@ export async function runTier3( const haveFts = !!input.ftsMatch && !!repos.worldModel.searchByText; const havePattern = !!input.patternTerms && input.patternTerms.length > 0 && !!repos.worldModel.searchByPattern; - if (!haveVec && !haveFts && !havePattern) return []; + const haveExact = + !!input.exactIdentifiers && + input.exactIdentifiers.length > 0 && + !!repos.worldModel.searchByPattern; + if (!haveVec && !haveFts && !havePattern && !haveExact) return []; const vecPoolSize = Math.max( config.tier3TopK, @@ -78,7 +82,6 @@ export async function runTier3( rank: idx, score: h.score, meta: h.meta, - vec: input.queryVec!, }); }); } @@ -91,7 +94,6 @@ export async function runTier3( rank: idx, score: h.score, meta: h.meta, - vec: input.queryVec ?? null, }); }); } @@ -104,7 +106,21 @@ export async function runTier3( rank: idx, score: h.score, meta: h.meta, - vec: input.queryVec ?? null, + }); + }); + } + if (haveExact) { + const hits = repos.worldModel.searchByPattern!( + input.exactIdentifiers!, + keywordPoolSize, + ); + hits.forEach((h, idx) => { + upsert(merged, h.id as WorldModelId, { + cosine: 0, + channel: "exact_identifier", + rank: idx, + score: 1, + meta: h.meta, }); }); } @@ -128,7 +144,7 @@ export async function runTier3( refId: wm.id, cosine: state.cosine, ts: Date.now(), - vec: state.vec, + vec: wm.vec ?? null, title: wm.title, body: wm.body, policyIds: wm.policyIds ?? [], @@ -142,7 +158,12 @@ export async function runTier3( log.info("done", { candidates: merged.size, kept: trimmed.length, - channels: { vec: haveVec, fts: haveFts, pattern: havePattern }, + channels: { + vec: haveVec, + fts: haveFts, + pattern: havePattern, + exactIdentifier: haveExact, + }, latencyMs: Date.now() - startedAt, }); return trimmed; @@ -164,7 +185,6 @@ function upsert( rank: number; score: number; meta?: { title: string }; - vec: EmbeddingVector | null; }, ): void { const entry = into.get(id); @@ -173,13 +193,11 @@ function upsert( cosine: patch.cosine, channels: [{ channel: patch.channel, rank: patch.rank, score: patch.score }], meta: patch.meta, - vec: patch.vec, }); return; } entry.channels.push({ channel: patch.channel, rank: patch.rank, score: patch.score }); if (patch.cosine > entry.cosine) entry.cosine = patch.cosine; - if (!entry.vec && patch.vec) entry.vec = patch.vec; } function bestChannelScore(c: WorldModelCandidate): number { diff --git a/apps/memos-local-plugin/core/retrieval/types.ts b/apps/memos-local-plugin/core/retrieval/types.ts index 19dca2f43..8ed24935e 100644 --- a/apps/memos-local-plugin/core/retrieval/types.ts +++ b/apps/memos-local-plugin/core/retrieval/types.ts @@ -36,6 +36,9 @@ import type { /** The three cost/benefit tiers; internal use only. */ export type TierKind = "tier1" | "tier2" | "tier3"; +/** Query-specific retrieval policy. Existing callers use `default`. */ +export type RetrievalProfile = "default" | "personal_fact"; + /** Which `vec_*` column on `traces` a Tier-2 hit came from. */ export type TraceVecKind = "summary" | "action"; @@ -56,6 +59,7 @@ export type RetrievalChannel = | "vec" | "fts" | "pattern" + | "exact_identifier" | "structural"; /** Per-channel rank carried on a candidate, used by the RRF pass. */ @@ -399,6 +403,7 @@ export interface RetrievalRepos { procedureJson?: unknown; decisionGuidance?: { preference: string[]; antiPattern: string[] }; eta: number; + vec: EmbeddingVector | null; sourcePolicyIds?: PolicyId[]; updatedAt?: EpochMs; } | null; @@ -542,6 +547,7 @@ export interface RetrievalRepos { title: string; body: string; policyIds: string[]; + vec: EmbeddingVector | null; } | null; }; @@ -704,6 +710,10 @@ export interface RetrievalStats { reason: RetrievalReason; /** Injection scheduler scenario, when turn-start routing was planned. */ scenarioId?: string; + /** Query-specific retrieval policy used by routing/ranking. */ + profile?: RetrievalProfile; + /** One normally; two when controlled widening ran. */ + retrievalPasses?: number; agent: AgentKind; sessionId: SessionId; episodeId?: EpisodeId; @@ -719,6 +729,8 @@ export interface RetrievalStats { totalLatencyMs: number; queryTokens: number; queryTags: string[]; + /** Count only; concrete identifiers are intentionally not logged. */ + exactIdentifierCount?: number; emptyPacket: boolean; /** Query embedding status. `degraded=true` means vector recall was unavailable. */ embedding?: { @@ -737,6 +749,11 @@ export interface RetrievalStats { */ rawCandidateCount?: number; droppedByThresholdCount?: number; + dedupedBeforeMmrCount?: number; + /** Capability variants merged after threshold eligibility was resolved. */ + dedupedAfterThresholdCount?: number; + /** Vector-only candidates rejected for long-identifier queries. */ + droppedByKeywordConfirmationCount?: number; thresholdFloor?: number; topRelevance?: number; rankedCount?: number; @@ -749,6 +766,7 @@ export interface RetrievalStats { | "deferred_to_final" | "llm_kept_all" | "llm_filtered" + | "llm_filtered_empty" | "llm_filtered_refilled" | "llm_failed_safe_cutoff"; llmFilterSufficient?: boolean; @@ -766,6 +784,7 @@ export interface RetrievalStats { | "vec" | "fts" | "pattern" + | "exact_identifier" | "structural", number >>; diff --git a/apps/memos-local-plugin/core/storage/keyword.ts b/apps/memos-local-plugin/core/storage/keyword.ts index 0b14fdf49..9d098a4cb 100644 --- a/apps/memos-local-plugin/core/storage/keyword.ts +++ b/apps/memos-local-plugin/core/storage/keyword.ts @@ -24,6 +24,16 @@ const TRIGRAM_MIN = 3; const PATTERN_MIN = 2; const MAX_FTS_TOKENS = 12; const MAX_PATTERN_TERMS = 16; +const MAX_EXACT_IDENTIFIERS = 8; +const MAX_EXACT_IDENTIFIER_CHARS = 128; +const EXACT_IDENTIFIER_TOKEN = /[A-Za-z0-9_:-]{12,}/g; +/** + * Characters that mainly express speaker, tense, modality or question + * scaffolding. A long-query bigram containing one of these is usually a + * conversational bridge ("你还", "得我", "什么") rather than a fact key. + * Exact two-character queries are still kept below, preserving names. + */ +const CJK_PATTERN_NOISE = /[我你他她它的了呢吗么还记得是有想请问谁哪帮]/u; export type FtsTokenizerMode = "trigram" | "cjk"; @@ -31,6 +41,32 @@ export interface PrepareFtsMatchOptions { tokenizer?: FtsTokenizerMode; } +/** + * Extract bounded high-entropy identifiers that require exact lexical + * confirmation. The concrete values are carried through retrieval but never + * added to structured logs. + */ +export function extractExactIdentifiers(query: string): string[] { + const tokens = String(query ?? "").match(EXACT_IDENTIFIER_TOKEN) ?? []; + const identifiers = new Set(); + for (const token of tokens) { + const normalized = normalizeIdentifierText(token); + if (normalized.length > MAX_EXACT_IDENTIFIER_CHARS) continue; + const hasIdentifierShape = /[_:-]/.test(normalized) || /\d/.test(normalized); + const hasEnoughEntropy = + /[a-z]/.test(normalized) && normalized.length >= 16; + if (!hasIdentifierShape || !hasEnoughEntropy) continue; + identifiers.add(normalized); + if (identifiers.size >= MAX_EXACT_IDENTIFIERS) break; + } + return [...identifiers]; +} + +/** Normalisation shared by query extraction and hydrated candidate checks. */ +export function normalizeIdentifierText(text: string): string { + return String(text ?? "").normalize("NFKC").toLowerCase().trim(); +} + /** * Sanitised FTS5 MATCH expression. * @@ -136,7 +172,8 @@ export function extractPatternTerms(query: string): string[] { continue; } for (let i = 0; i <= run.length - PATTERN_MIN; i++) { - out.add(run.slice(i, i + PATTERN_MIN)); + const term = run.slice(i, i + PATTERN_MIN); + if (!CJK_PATTERN_NOISE.test(term)) out.add(term); } } diff --git a/apps/memos-local-plugin/core/storage/repos/policies.ts b/apps/memos-local-plugin/core/storage/repos/policies.ts index 3a9b3390a..29920f60a 100644 --- a/apps/memos-local-plugin/core/storage/repos/policies.ts +++ b/apps/memos-local-plugin/core/storage/repos/policies.ts @@ -257,13 +257,16 @@ export function makePoliciesRepo(db: StorageDb) { k: Math.max(1, Math.min(200, Math.floor(k))), }; const ors: string[] = []; - dedup.slice(0, 16).forEach((t, i) => { + const matchScores: string[] = []; + const limitedTerms = dedup.slice(0, 16); + limitedTerms.forEach((t, i) => { const key = `pat_${i}`; const escaped = t.replace(/[\\%_]/g, (m) => `\\${m}`); params[key] = `%${escaped}%`; - ors.push( - `(title LIKE @${key} ESCAPE '\\' OR trigger LIKE @${key} ESCAPE '\\' OR procedure LIKE @${key} ESCAPE '\\' OR verification LIKE @${key} ESCAPE '\\' OR boundary LIKE @${key} ESCAPE '\\' OR decision_guidance_json LIKE @${key} ESCAPE '\\')`, - ); + const match = + `(title LIKE @${key} ESCAPE '\\' OR trigger LIKE @${key} ESCAPE '\\' OR procedure LIKE @${key} ESCAPE '\\' OR verification LIKE @${key} ESCAPE '\\' OR boundary LIKE @${key} ESCAPE '\\' OR decision_guidance_json LIKE @${key} ESCAPE '\\')`; + ors.push(match); + matchScores.push(`CASE WHEN ${match} THEN 1 ELSE 0 END`); }); const whereParts: string[] = [`(${ors.join(" OR ")})`]; if (opts.statusIn && opts.statusIn.length > 0) { @@ -285,17 +288,18 @@ export function makePoliciesRepo(db: StorageDb) { confidence, owner_agent_kind, owner_profile_id, - owner_workspace_id + owner_workspace_id, + (${matchScores.join(" + ")}) AS pattern_matches FROM policies WHERE ${whereParts.join(" AND ")} - ORDER BY updated_at DESC + ORDER BY pattern_matches DESC, updated_at DESC LIMIT @k`; const rows = db - .prepare(sql) + .prepare(sql) .all(params); - return rows.map((r, idx) => ({ + return rows.map((r) => ({ id: r.id, - score: 1 / (idx + 1), + score: r.pattern_matches / limitedTerms.length, meta: policySearchMeta(r), })); }, diff --git a/apps/memos-local-plugin/core/storage/repos/skills.ts b/apps/memos-local-plugin/core/storage/repos/skills.ts index db5a972e7..2fef2d71e 100644 --- a/apps/memos-local-plugin/core/storage/repos/skills.ts +++ b/apps/memos-local-plugin/core/storage/repos/skills.ts @@ -251,13 +251,16 @@ export function makeSkillsRepo(db: StorageDb) { k: Math.max(1, Math.min(200, Math.floor(k))), }; const ors: string[] = []; - dedup.slice(0, 16).forEach((t, i) => { + const matchScores: string[] = []; + const limitedTerms = dedup.slice(0, 16); + limitedTerms.forEach((t, i) => { const key = `pat_${i}`; const escaped = t.replace(/[\\%_]/g, (m) => `\\${m}`); params[key] = `%${escaped}%`; - ors.push( - `(name LIKE @${key} ESCAPE '\\' OR invocation_guide LIKE @${key} ESCAPE '\\')`, - ); + const match = + `(name LIKE @${key} ESCAPE '\\' OR invocation_guide LIKE @${key} ESCAPE '\\')`; + ors.push(match); + matchScores.push(`CASE WHEN ${match} THEN 1 ELSE 0 END`); }); const whereParts: string[] = [`(${ors.join(" OR ")})`]; if (opts.statusIn && opts.statusIn.length > 0) { @@ -268,17 +271,18 @@ export function makeSkillsRepo(db: StorageDb) { }); } const sql = ` - SELECT id, name, status, eta, gain, owner_agent_kind, owner_profile_id, owner_workspace_id + SELECT id, name, status, eta, gain, owner_agent_kind, owner_profile_id, + owner_workspace_id, (${matchScores.join(" + ")}) AS pattern_matches FROM skills WHERE ${whereParts.join(" AND ")} - ORDER BY updated_at DESC + ORDER BY pattern_matches DESC, updated_at DESC LIMIT @k`; const rows = db - .prepare(sql) + .prepare(sql) .all(params); - return rows.map((r, idx) => ({ + return rows.map((r) => ({ id: r.id, - score: 1 / (idx + 1), + score: r.pattern_matches / limitedTerms.length, meta: { name: r.name, status: r.status, eta: r.eta, gain: r.gain, owner_agent_kind: r.owner_agent_kind, owner_profile_id: r.owner_profile_id, owner_workspace_id: r.owner_workspace_id }, })); }, diff --git a/apps/memos-local-plugin/core/storage/repos/traces.ts b/apps/memos-local-plugin/core/storage/repos/traces.ts index 894552b56..d78094872 100644 --- a/apps/memos-local-plugin/core/storage/repos/traces.ts +++ b/apps/memos-local-plugin/core/storage/repos/traces.ts @@ -650,33 +650,36 @@ export function makeTracesRepo(db: StorageDb) { k: Math.max(1, Math.min(500, Math.floor(k))), }; const ors: string[] = []; - dedup.slice(0, 16).forEach((t, i) => { + const matchScores: string[] = []; + const limitedTerms = dedup.slice(0, 16); + limitedTerms.forEach((t, i) => { const key = `pat_${i}`; // Escape SQL LIKE wildcards in the user term so a literal `%` // doesn't accidentally match everything. const escaped = t.replace(/[\\%_]/g, (m) => `\\${m}`); params[key] = `%${escaped}%`; - ors.push( - `(user_text LIKE @${key} ESCAPE '\\' OR + const match = `(user_text LIKE @${key} ESCAPE '\\' OR agent_text LIKE @${key} ESCAPE '\\' OR COALESCE(summary,'') LIKE @${key} ESCAPE '\\' OR COALESCE(reflection,'') LIKE @${key} ESCAPE '\\' OR - tags_json LIKE @${key} ESCAPE '\\')`, - ); + tags_json LIKE @${key} ESCAPE '\\')`; + ors.push(match); + matchScores.push(`CASE WHEN ${match} THEN 1 ELSE 0 END`); }); const extra = opts.where ? ` AND (${opts.where})` : ""; const sql = ` SELECT id, ts, priority, value, episode_id, session_id, tags_json, owner_agent_kind, owner_profile_id, owner_workspace_id, - error_signatures_json + error_signatures_json, + (${matchScores.join(" + ")}) AS pattern_matches FROM traces WHERE (${ors.join(" OR ")})${extra} - ORDER BY ts DESC + ORDER BY pattern_matches DESC, ts DESC LIMIT @k`; - const rows = db.prepare(sql).all(params); - return rows.map((r, idx) => ({ + const rows = db.prepare(sql).all(params); + return rows.map((r) => ({ id: r.id, - score: 1 / (idx + 1), + score: r.pattern_matches / limitedTerms.length, meta: { ts: r.ts, priority: r.priority, diff --git a/apps/memos-local-plugin/core/storage/repos/world_model.ts b/apps/memos-local-plugin/core/storage/repos/world_model.ts index f56939308..6a3a95662 100644 --- a/apps/memos-local-plugin/core/storage/repos/world_model.ts +++ b/apps/memos-local-plugin/core/storage/repos/world_model.ts @@ -230,30 +230,34 @@ export function makeWorldModelRepo(db: StorageDb) { k: Math.max(1, Math.min(200, Math.floor(k))), }; const ors: string[] = []; - dedup.slice(0, 16).forEach((t, i) => { + const matchScores: string[] = []; + const limitedTerms = dedup.slice(0, 16); + limitedTerms.forEach((t, i) => { const key = `pat_${i}`; const escaped = t.replace(/[\\%_]/g, (m) => `\\${m}`); params[key] = `%${escaped}%`; - ors.push( - `(title LIKE @${key} ESCAPE '\\' OR body LIKE @${key} ESCAPE '\\' OR COALESCE(domain_tags_json,'') LIKE @${key} ESCAPE '\\')`, - ); + const match = + `(title LIKE @${key} ESCAPE '\\' OR body LIKE @${key} ESCAPE '\\' OR COALESCE(domain_tags_json,'') LIKE @${key} ESCAPE '\\')`; + ors.push(match); + matchScores.push(`CASE WHEN ${match} THEN 1 ELSE 0 END`); }); const whereParts: string[] = [`(${ors.join(" OR ")})`]; if (opts.minConfidence !== undefined) { whereParts.push(`confidence >= ${Number(opts.minConfidence)}`); } const sql = ` - SELECT id, title, owner_agent_kind, owner_profile_id, owner_workspace_id + SELECT id, title, owner_agent_kind, owner_profile_id, owner_workspace_id, + (${matchScores.join(" + ")}) AS pattern_matches FROM world_model WHERE ${whereParts.join(" AND ")} - ORDER BY updated_at DESC + ORDER BY pattern_matches DESC, updated_at DESC LIMIT @k`; const rows = db - .prepare(sql) + .prepare(sql) .all(params); - return rows.map((r, idx) => ({ + return rows.map((r) => ({ id: r.id, - score: 1 / (idx + 1), + score: r.pattern_matches / limitedTerms.length, meta: { title: r.title, owner_agent_kind: r.owner_agent_kind, owner_profile_id: r.owner_profile_id, owner_workspace_id: r.owner_workspace_id }, })); }, diff --git a/apps/memos-local-plugin/tests/unit/injection/scheduler.test.ts b/apps/memos-local-plugin/tests/unit/injection/scheduler.test.ts index 8b113b42d..5f17c02bf 100644 --- a/apps/memos-local-plugin/tests/unit/injection/scheduler.test.ts +++ b/apps/memos-local-plugin/tests/unit/injection/scheduler.test.ts @@ -11,9 +11,13 @@ const baseIntent: IntentDecision = { signals: ["test"], }; -function planFor(intent: IntentDecision, relation?: Parameters[0]["relation"]) { +function planFor( + intent: IntentDecision, + relation?: Parameters[0]["relation"], + userText = "test", +) { return scheduleInjection({ - userText: "test", + userText, sessionId: "s1", episodeId: "ep1", intent, @@ -77,6 +81,46 @@ describe("injection/scheduler", () => { expect(plan).toMatchObject({ scenarioId: "MEMORY_PROBE", entry: "turn_start", + profile: "default", + wantTier1: true, + wantTier2: true, + wantTier3: false, + }); + }); + + it("routes high-confidence personal-fact probes to Tier 2 only", () => { + const plan = planFor( + { + ...baseIntent, + kind: "memory_probe", + retrieval: { tier1: true, tier2: true, tier3: false }, + }, + undefined, + "你还记得我喜欢什么吗?", + ); + + expect(plan).toMatchObject({ + scenarioId: "MEMORY_PROBE", + profile: "personal_fact", + wantTier1: false, + wantTier2: true, + wantTier3: false, + }); + }); + + it("does not misroute remembered troubleshooting as a personal fact", () => { + const plan = planFor( + { + ...baseIntent, + kind: "memory_probe", + retrieval: { tier1: true, tier2: true, tier3: false }, + }, + undefined, + "你还记得上次 OpenClaw Gateway 是怎么修复的吗?", + ); + + expect(plan).toMatchObject({ + profile: "default", wantTier1: true, wantTier2: true, wantTier3: false, diff --git a/apps/memos-local-plugin/tests/unit/pipeline/retrieval-log.test.ts b/apps/memos-local-plugin/tests/unit/pipeline/retrieval-log.test.ts new file mode 100644 index 000000000..d8ae9cdc0 --- /dev/null +++ b/apps/memos-local-plugin/tests/unit/pipeline/retrieval-log.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it } from "vitest"; + +import type { InjectionSnippet } from "../../../agent-contract/dto.js"; +import { buildLocalRetrievalLogStages } from "../../../core/pipeline/retrieval-log.js"; + +function snippet(refId: string, score: number): InjectionSnippet { + return { + refKind: "trace", + refId, + body: `memory ${refId}`, + score, + }; +} + +describe("pipeline/retrieval-log", () => { + it("reconstructs the pre-LLM candidates from kept and dropped snippets", () => { + // The LLM is allowed to rerank kept candidates independently of the + // mechanical score. The initial funnel must recover score order without + // changing the actual post-LLM injection order or losing dropped rows. + const kept = [snippet("kept-direct-answer", 0.21), snippet("kept-top-score", 1.013)]; + const dropped = [snippet("dropped-middle", 0.4)]; + + const stages = buildLocalRetrievalLogStages({ + snippets: kept, + droppedByLlm: dropped, + }); + + expect(stages.candidates).toHaveLength(kept.length + dropped.length); + expect(stages.candidates.map((candidate) => candidate.refId)).toEqual([ + "kept-top-score", + "dropped-middle", + "kept-direct-answer", + ]); + expect(stages.filtered.map((candidate) => candidate.refId)).toEqual([ + "kept-direct-answer", + "kept-top-score", + ]); + expect(stages.dropped.map((candidate) => candidate.refId)).toEqual(["dropped-middle"]); + expect(stages.dropped[0]?.score).toBe(0.4); + }); + + it("preserves score composition for candidate diagnostics", () => { + const item = snippet("fact", 0.72); + item.scoreDetails = { + profile: "personal_fact", + semantic: 0.82, + tierBoost: 0, + rrfBoost: 0.01, + relevance: 0.83, + mmrLambda: 0.85, + redundancy: 0.2, + finalScore: 0.72, + channels: ["vec_summary"], + bypassedThreshold: false, + }; + + const stages = buildLocalRetrievalLogStages({ snippets: [item] }); + + expect(stages.filtered[0]?.scoreDetails).toEqual(item.scoreDetails); + }); +}); diff --git a/apps/memos-local-plugin/tests/unit/retrieval/integration.test.ts b/apps/memos-local-plugin/tests/unit/retrieval/integration.test.ts index 09e607c92..7bd8c85fa 100644 --- a/apps/memos-local-plugin/tests/unit/retrieval/integration.test.ts +++ b/apps/memos-local-plugin/tests/unit/retrieval/integration.test.ts @@ -245,6 +245,69 @@ describe("retrieval/integration", () => { expect(refKinds).toContain("world-model"); expect(refKinds).not.toContain("trace"); expect(refKinds).not.toContain("episode"); + expect(res.stats.droppedByKeywordConfirmationCount).toBeGreaterThan(0); + }); + + it("recalls the exact identifier trace and rejects a generic pattern impostor", async () => { + const identifier = "project_id_2026_alpha_001"; + const insertIdentifierTrace = (input: { + id: string; + userText: string; + }) => { + handle.repos.traces.insert({ + id: input.id as TraceId, + episodeId: "ep1" as EpisodeId, + sessionId: "s1" as SessionId, + ts: NOW as never, + userText: input.userText, + agentText: "状态已确认", + toolCalls: [], + reflection: null, + value: 0.8 as never, + alpha: 0.5 as never, + rHuman: null, + priority: 0.8 as never, + tags: [], + vecSummary: vec([1, 0, 0]), + vecAction: null, + turnId: 0 as never, + schemaVersion: 1, + }); + }; + insertIdentifierTrace({ + id: "trace_exact_identifier", + userText: `项目 ${identifier} 已完成`, + }); + insertIdentifierTrace({ + id: "trace_generic_pattern", + userText: "请查询之前的普通项目记录", + }); + + const base = makeDeps(handle); + const res = await toolDrivenRetrieve( + { + ...base, + config: { + ...base.config, + lightweightMemory: true, + }, + }, + { + reason: "tool_driven", + agent: "openclaw", + sessionId: "s1" as SessionId, + tool: "memos_search", + args: { query: `请查询 ${identifier} 的记录` }, + ts: NOW as never, + }, + ); + + const ids = res.packet.snippets.map((snippet) => String(snippet.refId)); + expect(ids).toContain("trace_exact_identifier"); + expect(ids).not.toContain("trace_generic_pattern"); + expect(res.stats.exactIdentifierCount).toBe(1); + expect(res.stats.channelHits?.exact_identifier).toBeGreaterThan(0); + expect(res.stats.droppedByKeywordConfirmationCount).toBeGreaterThan(0); }); it("recalls feedback experiences through keyword channels when embeddings degrade", async () => { @@ -335,6 +398,10 @@ describe("retrieval/integration", () => { expect(res.stats.llmFilterOutcome).toBe("llm_filtered"); expect(res.stats.emptyPacket).toBe(false); expect(filterCalls).toBe(1); + expect(res.packet.droppedByLlm?.length).toBeGreaterThan(0); + expect(res.packet.droppedByLlm?.every((snippet) => typeof snippet.score === "number")).toBe( + true, + ); }); it("can defer the local LLM pass for one final merged filter", async () => { @@ -402,14 +469,7 @@ describe("retrieval/integration", () => { expect(res.stats.emptyPacket).toBe(false); }); - it("turn_start rescues injection when LLM filter empties the kept set (#1913)", async () => { - // Repro for issue #1913: when the LLM relevance filter returns - // `selected: []` for a non-empty ranked list (the case where the - // top hits are all near-duplicate question traces), the packet - // used to collapse to an empty injection. The rescue path keeps - // the top-K best-scoring candidates so the agent still gets a - // packet, and surfaces `llm_filtered_refilled` so the Logs viewer - // can show the safety net fired. + it("turn_start honours an empty LLM selection instead of injecting unrelated candidates", async () => { const llm: any = { completeJson: async () => ({ value: { selected: [], sufficient: false }, @@ -435,10 +495,85 @@ describe("retrieval/integration", () => { }, ); - expect(res.packet.snippets.length).toBeGreaterThan(0); - expect(res.packet.rendered.length).toBeGreaterThan(0); - expect(res.stats.llmFilterOutcome).toBe("llm_filtered_refilled"); - expect(res.stats.emptyPacket).toBe(false); + expect(res.packet.snippets).toEqual([]); + expect(res.packet.rendered).toBe(""); + expect(res.stats.llmFilterOutcome).toBe("llm_filtered_empty"); + expect(res.stats.emptyPacket).toBe(true); + }); + + it("widens a personal-fact query to the action channel only after an empty LLM pass", async () => { + handle.repos.traces.insert({ + id: "t_personal_action" as TraceId, + episodeId: "ep1" as EpisodeId, + sessionId: "s1" as SessionId, + ts: NOW as never, + userText: "我最偏爱的歌手是陈奕迅", + agentText: "已记录这条个人偏好。", + toolCalls: [], + summary: null, + reflection: null, + value: 0.7 as never, + alpha: 0.5 as never, + rHuman: null, + priority: 0.7 as never, + tags: [], + vecSummary: vec([0, 1, 0]), + vecAction: vec([1, 0, 0]), + turnId: 0 as never, + schemaVersion: 1, + }); + let filterCalls = 0; + const llm: any = { + completeJson: async (messages: Array<{ role: string; content: string }>) => { + filterCalls += 1; + if (filterCalls === 1) { + return { + value: { selected: [], sufficient: false }, + servedBy: "fake", + }; + } + const candidateText = messages[1]!.content; + const match = candidateText.match(/^(\d+)\..*陈奕迅.*$/m); + return { + value: { selected: [Number(match?.[1] ?? 0)], sufficient: true }, + servedBy: "fake", + }; + }, + }; + + const res = await turnStartRetrieve( + { + ...makeDeps(handle), + llm, + config: { + ...makeDeps(handle).config, + llmFilterEnabled: true, + llmFilterMinCandidates: 1, + }, + }, + { + reason: "turn_start", + agent: "openclaw", + sessionId: "s_current" as SessionId, + userText: "你还记得关于我的偏好吗?", + ts: NOW as never, + }, + { + plan: { + profile: "personal_fact", + wantTier1: false, + wantTier2: true, + wantTier3: false, + }, + }, + ); + + expect(filterCalls).toBe(2); + expect(res.stats.retrievalPasses).toBe(2); + expect(res.packet.snippets.map((snippet) => snippet.body).join("\n")).toContain( + "陈奕迅", + ); + expect(res.packet.snippets[0]?.scoreDetails?.channels).toContain("vec_action"); }); it("skill_invoke is tier1-heavy", async () => { diff --git a/apps/memos-local-plugin/tests/unit/retrieval/llm-filter.test.ts b/apps/memos-local-plugin/tests/unit/retrieval/llm-filter.test.ts index 2355732d7..a69a3d954 100644 --- a/apps/memos-local-plugin/tests/unit/retrieval/llm-filter.test.ts +++ b/apps/memos-local-plugin/tests/unit/retrieval/llm-filter.test.ts @@ -57,33 +57,6 @@ function trace(id: string, score: number): RankedCandidate { }; } -/** - * Build a trace candidate with no LLM-generated summary / reflection - * and only a short acknowledgement-style `agentText` — i.e. exactly the - * "near-duplicate question trace" shape from issue #1913 where the - * user re-asked a stored fact across multiple sessions and the - * assistant only acked it. The current LLM filter prompt classes these - * as "scaffolding chatter" and is allowed to drop them all. - */ -function chatterTrace( - id: string, - score: number, - overrides: { agentText?: string; userText?: string } = {}, -): RankedCandidate { - const r = trace(id, score); - const cand = r.candidate as TraceCandidate; - return { - ...r, - candidate: { - ...cand, - userText: overrides.userText ?? `What does HERMES_REAL_E2E_1910 mean? (${id})`, - agentText: overrides.agentText ?? "OK", - summary: null, - reflection: null, - }, - }; -} - describe("retrieval/llm-filter", () => { it("disabled → passthrough with null sufficient", async () => { const result = await llmFilterCandidates( @@ -162,100 +135,47 @@ describe("retrieval/llm-filter", () => { expect(result.sufficient).toBe(true); }); - it("LLM returns empty selection over non-empty ranked → rescues top-K informative candidates (#1913)", async () => { - // Issue #1913 repro shape: 3 near-duplicate question traces from - // previous sessions plus 1 answer-bearing trace. Previous filter - // honoured `selected: []` and collapsed to empty injection. + it("honours an empty LLM selection instead of mechanically refilling candidates", async () => { const llm: any = { completeJson: vi.fn().mockResolvedValue({ value: { selected: [], sufficient: false }, servedBy: "fake", }), }; - const ranked = [ - chatterTrace("q1", 0.95), - chatterTrace("q2", 0.94), - // Answer-bearing trace: long informative agentText, even though - // the ranker placed it below the question duplicates. - (() => { - const r = trace("answer", 0.85); - const c = r.candidate as TraceCandidate; - return { - ...r, - candidate: { - ...c, - userText: - "Remember this fact: HERMES_REAL_E2E_1910 means the bridge leak fix verification.", - agentText: "Noted. HERMES_REAL_E2E_1910 → bridge leak fix verification.", - summary: - "HERMES_REAL_E2E_1910 marks the bridge-leak fix verification.", - reflection: null, - }, - }; - })(), - chatterTrace("q3", 0.8), - ]; + const ranked = [trace("unrelated-a", 0.9), trace("unrelated-b", 0.8)]; + const result = await llmFilterCandidates( - { query: "What does HERMES_REAL_E2E_1910 mean?", ranked }, + { query: "好的,先做成ppt文件,我一会填数据", ranked }, { llm, log, config: cfg }, ); - expect(result.outcome).toBe("llm_filtered_refilled"); - expect(result.kept.length).toBeGreaterThanOrEqual(1); - expect(result.kept[0]!.candidate.refId).toBe("answer"); + + expect(result.outcome).toBe("llm_filtered_empty"); + expect(result.kept).toEqual([]); + expect(result.dropped).toEqual(ranked); expect(result.sufficient).toBe(false); - // Strong negative assertion: the bug returned kept=[], dropped=ranked. - expect(result.dropped.length).toBeLessThan(ranked.length); }); - it("rescue fires even when every ranked candidate is short-ack chatter", async () => { + it("honours an empty LLM selection for personal facts so recall can widen safely", async () => { const llm: any = { completeJson: vi.fn().mockResolvedValue({ value: { selected: [], sufficient: false }, servedBy: "fake", }), }; - const ranked = [ - chatterTrace("q1", 0.9, { agentText: "OK" }), - chatterTrace("q2", 0.85, { agentText: "记住了" }), - chatterTrace("q3", 0.8, { agentText: "👍" }), - ]; + const ranked = [trace("unrelated-a", 0.9), trace("unrelated-b", 0.8)]; + const result = await llmFilterCandidates( - { query: "What does HERMES_REAL_E2E_1910 mean?", ranked }, + { query: "我喜欢什么?", ranked, profile: "personal_fact" }, { llm, log, config: cfg }, ); - expect(result.outcome).toBe("llm_filtered_refilled"); - // No informative candidate exists — rescue still keeps top-K by - // ranker score so the agent at least sees one memory. - expect(result.kept.length).toBeGreaterThanOrEqual(1); - expect(result.kept[0]!.candidate.refId).toBe("q1"); // highest score - expect(result.sufficient).toBe(false); - }); - it("rescue respects llmFilterMaxKeep cap", async () => { - const llm: any = { - completeJson: vi.fn().mockResolvedValue({ - value: { selected: [], sufficient: false }, - servedBy: "fake", - }), - }; - const ranked = [ - trace("a", 0.95), - trace("b", 0.9), - trace("c", 0.85), - trace("d", 0.8), - ]; - const result = await llmFilterCandidates( - { query: "q", ranked }, - { llm, log, config: { ...cfg, llmFilterMaxKeep: 2 } }, - ); - expect(result.outcome).toBe("llm_filtered_refilled"); - expect(result.kept.length).toBe(2); - expect(result.dropped.length).toBe(2); + expect(result.outcome).toBe("llm_filtered_empty"); + expect(result.kept).toEqual([]); + expect(result.dropped).toEqual(ranked); + expect(result.sufficient).toBe(false); }); - it("llmFilterMaxKeep=0 disables rescue: honours empty selection (drop-everything override)", async () => { - // This is the only configured way to ask the filter to truly drop - // everything; keep it explicit so operators have an escape hatch. + it("llmFilterMaxKeep=0 honours an empty selection", async () => { const llm: any = { completeJson: vi.fn().mockResolvedValue({ value: { selected: [], sufficient: false }, @@ -267,7 +187,7 @@ describe("retrieval/llm-filter", () => { { query: "q", ranked }, { llm, log, config: { ...cfg, llmFilterMaxKeep: 0 } }, ); - expect(result.outcome).toBe("llm_filtered"); + expect(result.outcome).toBe("llm_filtered_empty"); expect(result.kept.length).toBe(0); expect(result.dropped.length).toBe(2); expect(result.sufficient).toBe(false); diff --git a/apps/memos-local-plugin/tests/unit/retrieval/query-builder.test.ts b/apps/memos-local-plugin/tests/unit/retrieval/query-builder.test.ts index f5b55cf37..02cbc51a7 100644 --- a/apps/memos-local-plugin/tests/unit/retrieval/query-builder.test.ts +++ b/apps/memos-local-plugin/tests/unit/retrieval/query-builder.test.ts @@ -6,22 +6,69 @@ import type { EpochMs } from "../../../core/types.js"; const NOW = 1_700_000_000_000 as EpochMs; describe("retrieval/query-builder", () => { - it("turn_start embeds userText + hints", () => { + it("turn_start embeds only userText and excludes routing contextHints", () => { const cq = buildQuery({ reason: "turn_start", agent: "openclaw", sessionId: "s1" as unknown as never, userText: "Fix this docker compose file", - contextHints: { cwd: "/tmp/x", role: "planner" }, + contextHints: { + cwd: "/tmp/x", + role: "planner", + message_id: "om_x100b69aac421a4a4ddbedba7b69bbb6", + sender_id: "ou_7ca281bf4db7089a424187d677f3cfa3", + }, ts: NOW, }); - expect(cq.text).toContain("Fix this docker compose file"); - expect(cq.text).toContain("cwd: /tmp/x"); - expect(cq.text).toContain("role: planner"); + expect(cq.text).toBe("Fix this docker compose file"); + expect(cq.text).not.toContain("cwd"); + expect(cq.text).not.toContain("message_id"); + expect(cq.text).not.toContain("sender_id"); expect(cq.tags).toContain("docker"); + expect(cq.exactIdentifiers).toEqual([]); expect(cq.truncated).toBe(false); }); + it("extracts bounded long identifiers without promoting ordinary pattern terms", () => { + const cq = buildQuery({ + reason: "turn_start", + agent: "openclaw", + sessionId: "s1" as unknown as never, + userText: "请查询 project_id_2026_alpha_001 的记录", + ts: NOW, + }); + + expect(cq.exactIdentifiers).toEqual(["project_id_2026_alpha_001"]); + expect(cq.patternTerms).toEqual(["查询"]); + }); + + it("does not extract long identifiers from routing context hints", () => { + const cq = buildQuery({ + reason: "turn_start", + agent: "openclaw", + sessionId: "s1" as unknown as never, + userText: "我喜欢芒果", + contextHints: { + message_id: "om_x100b69aac421a4a4ddbedba7b69bbb6", + }, + ts: NOW, + }); + + expect(cq.exactIdentifiers).toEqual([]); + }); + + it("rejects oversized identifiers instead of confirming a truncated prefix", () => { + const cq = buildQuery({ + reason: "turn_start", + agent: "openclaw", + sessionId: "s1" as unknown as never, + userText: `查询 project_${"a".repeat(200)}`, + ts: NOW, + }); + + expect(cq.exactIdentifiers).toEqual([]); + }); + it("tool_driven uses explicit search query text when present", () => { const cq = buildQuery({ reason: "tool_driven", diff --git a/apps/memos-local-plugin/tests/unit/retrieval/ranker.test.ts b/apps/memos-local-plugin/tests/unit/retrieval/ranker.test.ts index 4121559f1..6f3d24c95 100644 --- a/apps/memos-local-plugin/tests/unit/retrieval/ranker.test.ts +++ b/apps/memos-local-plugin/tests/unit/retrieval/ranker.test.ts @@ -62,8 +62,8 @@ function trace(id: string, cos: number, value: number, vec?: number[]): TraceCan episodeId: `ep-${id}` as never, sessionId: "s1" as never, vecKind: "summary", - userText: "u", - agentText: "a", + userText: `user ${id}`, + agentText: `agent ${id}`, summary: null, reflection: null, tags: [], @@ -164,6 +164,410 @@ describe("retrieval/ranker", () => { expect(picked).toContain("diff"); }); + it("hard-dedupes repeated-question chatter before MMR so Top-K is refilled", () => { + const question = "我最喜欢的水果是什么?"; + const ackHigh = trace("ack_high", 0.99, 0, [1, 0]); + const ackSecond = trace("ack_second", 0.98, 0, [1, 0]); + const answer = trace("answer", 0.95, 0, [1, 0]); + const other = trace("other", 0.7, 0, [0, 1]); + Object.assign(ackHigh, { userText: question, agentText: "好的" }); + Object.assign(ackSecond, { userText: ` ${question}!`, agentText: "收到" }); + Object.assign(answer, { + userText: question, + agentText: "你之前明确说过,自己最喜欢的水果一直都是芒果。", + }); + Object.assign(other, { + userText: "我最喜欢的歌手是谁?", + agentText: "你之前提到过,自己最喜欢的歌手一直都是陈奕迅。", + }); + + const out = rank({ + tier1: [], + tier2Traces: [ackHigh, ackSecond, answer, other], + tier2Episodes: [], + tier3: [], + limit: 2, + profile: "personal_fact", + config: { + ...cfg, + relativeThresholdFloor: 0, + smartSeed: false, + }, + now: NOW, + }); + + expect(out.ranked.map((item) => String(item.candidate.refId))).toEqual([ + "answer", + "other", + ]); + expect(out.dedupedBeforeMmr).toBe(2); + }); + + it("dedupes before recomputing the relative threshold so chatter cannot raise the floor", () => { + const question = "我最喜欢的水果是什么?"; + const highChatter = trace("high_chatter", 0.99, 0); + const answer = trace("answer", 0.75, 0); + const other = trace("other", 0.74, 0); + Object.assign(highChatter, { + userText: question, + agentText: "好的", + }); + Object.assign(answer, { + userText: question, + agentText: "你之前明确说过,自己最喜欢的水果一直都是芒果。", + }); + Object.assign(other, { + userText: "我最喜欢的歌手是谁?", + agentText: "你之前提到过,自己最喜欢的歌手一直都是陈奕迅。", + }); + + const out = rank({ + tier1: [], + tier2Traces: [highChatter, answer, other], + tier2Episodes: [], + tier3: [], + limit: 2, + config: { + ...cfg, + relativeThresholdFloor: 0.8, + smartSeed: false, + }, + now: NOW, + }); + + expect(out.ranked.map((item) => String(item.candidate.refId))).toEqual([ + "answer", + "other", + ]); + expect(out.dedupedBeforeMmr).toBe(1); + expect(out.topRelevance).toBeCloseTo(0.75, 5); + expect(out.thresholdFloor).toBeCloseTo(0.6, 5); + }); + + it("uses post-score relevance to choose an exact-summary duplicate before MMR", () => { + const question = "我最喜欢的水果是什么?"; + const lower = trace("lower", 0.89, 0); + const higher = trace("higher", 0.9, 0); + const other = trace("other", 0.7, 0); + Object.assign(lower, { + userText: question, + agentText: "好的", + summary: "用户最喜欢的水果是芒果", + }); + Object.assign(higher, { + userText: `${question}!`, + agentText: "收到", + summary: "用户最喜欢的水果是芒果", + }); + + const out = rank({ + tier1: [], + tier2Traces: [lower, higher, other], + tier2Episodes: [], + tier3: [], + limit: 2, + config: { + ...cfg, + relativeThresholdFloor: 0, + smartSeed: false, + }, + now: NOW, + }); + + expect(out.ranked.map((item) => String(item.candidate.refId))).toEqual([ + "higher", + "other", + ]); + expect(out.dedupedBeforeMmr).toBe(1); + }); + + it("rejects a generic pattern impostor and keeps the candidate containing the exact identifier", () => { + const identifier = "project_id_2026_alpha_001"; + const patternImpostor = trace("pattern_impostor", 0.9, 0); + const exactMatch = trace("exact_match", 0.8, 0); + Object.assign(patternImpostor, { + userText: "请查询之前的普通项目记录", + agentText: "好的", + summary: "项目状态为已完成", + channels: [ + { channel: "vec_summary", rank: 0, score: 0.9 }, + { channel: "pattern", rank: 0, score: 1 }, + ], + }); + Object.assign(exactMatch, { + userText: `项目 ${identifier} 的状态`, + agentText: "收到", + summary: "项目状态为已完成", + channels: [{ channel: "exact_identifier", rank: 0, score: 1 }], + }); + + const out = rank({ + tier1: [], + tier2Traces: [patternImpostor, exactMatch], + tier2Episodes: [], + tier3: [], + limit: 1, + exactIdentifiers: [identifier], + config: { + ...cfg, + relativeThresholdFloor: 0, + smartSeed: false, + }, + now: NOW, + }); + + expect(out.ranked.map((item) => String(item.candidate.refId))).toEqual([ + "exact_match", + ]); + expect(out.droppedByKeywordConfirmation).toBe(1); + }); + + it("does not let a higher single-channel duplicate erase a lower multi-channel bypass candidate", () => { + const question = "我最喜欢的水果是什么?"; + const top = trace("top", 1, 0); + const ordinary = trace("ordinary", 0.58, 0); + const bypass = trace("bypass", 0.55, 0); + Object.assign(top, { + channels: [{ channel: "vec_summary", rank: 0, score: 1 }], + }); + Object.assign(ordinary, { + userText: question, + agentText: "好的", + summary: "用户最喜欢的水果是芒果", + channels: [{ channel: "vec_summary", rank: 1, score: 0.58 }], + }); + Object.assign(bypass, { + userText: question, + agentText: "收到", + summary: "用户最喜欢的水果是芒果", + channels: [ + { channel: "vec_summary", rank: 5, score: 0.55 }, + { channel: "pattern", rank: 4, score: 0.2 }, + ], + }); + + const out = rank({ + tier1: [], + tier2Traces: [top, ordinary, bypass], + tier2Episodes: [], + tier3: [], + limit: 3, + config: { + ...cfg, + relativeThresholdFloor: 0.6, + multiChannelBypass: true, + smartSeed: false, + }, + now: NOW, + }); + + const ids = out.ranked.map((item) => String(item.candidate.refId)); + expect(ids).toContain("top"); + expect(ids).toContain("bypass"); + expect(ids).not.toContain("ordinary"); + expect(out.ranked.find((item) => item.candidate.refId === bypass.refId)) + .toMatchObject({ bypassedThreshold: true }); + }); + + it("collapses capability variants again after both duplicates survive the threshold", () => { + const question = "我最喜欢的水果是什么?"; + const ordinary = trace("ordinary", 0.8, 0); + const bypassCapable = trace("bypass_capable", 0.75, 0); + Object.assign(ordinary, { + userText: question, + agentText: "好的", + summary: "用户最喜欢的水果是芒果", + channels: [{ channel: "vec_summary", rank: 0, score: 0.8 }], + }); + Object.assign(bypassCapable, { + userText: question, + agentText: "收到", + summary: "用户最喜欢的水果是芒果", + channels: [ + { channel: "vec_summary", rank: 1, score: 0.75 }, + { channel: "pattern", rank: 0, score: 0.2 }, + ], + }); + + const out = rank({ + tier1: [], + tier2Traces: [ordinary, bypassCapable], + tier2Episodes: [], + tier3: [], + limit: 2, + config: { + ...cfg, + relativeThresholdFloor: 0.4, + multiChannelBypass: true, + smartSeed: false, + }, + now: NOW, + }); + + expect(out.ranked.map((item) => String(item.candidate.refId))).toEqual([ + "ordinary", + ]); + expect(out.dedupedBeforeMmr).toBe(1); + expect(out.dedupedAfterThreshold).toBe(1); + }); + + it("keeps different concise fact answers to the same question", () => { + const question = "我最喜欢的水果是什么?"; + const oldAnswer = trace("old_short", 0.9, 0); + const newAnswer = trace("new_short", 0.85, 0); + Object.assign(oldAnswer, { + userText: question, + agentText: "苹果", + }); + Object.assign(newAnswer, { + userText: question, + agentText: "芒果", + }); + + const out = rank({ + tier1: [], + tier2Traces: [oldAnswer, newAnswer], + tier2Episodes: [], + tier3: [], + limit: 2, + config: { + ...cfg, + relativeThresholdFloor: 0, + smartSeed: false, + }, + now: NOW, + }); + + expect(out.ranked.map((item) => String(item.candidate.refId))).toEqual([ + "old_short", + "new_short", + ]); + expect(out.dedupedBeforeMmr).toBe(0); + }); + + it("preserves semantic symbols when grouping repeated questions", () => { + const cpp = trace("cpp", 0.9, 0); + const c = trace("c", 0.85, 0); + Object.assign(cpp, { + userText: "C++ 怎么学?", + agentText: "好的", + }); + Object.assign(c, { + userText: "C 怎么学?", + agentText: "收到", + }); + + const out = rank({ + tier1: [], + tier2Traces: [cpp, c], + tier2Episodes: [], + tier3: [], + limit: 2, + config: { + ...cfg, + relativeThresholdFloor: 0, + smartSeed: false, + }, + now: NOW, + }); + + expect(out.ranked.map((item) => String(item.candidate.refId))).toEqual([ + "cpp", + "c", + ]); + expect(out.dedupedBeforeMmr).toBe(0); + }); + + it("keeps a summary-only fact alongside a different substantive answer", () => { + const question = "我最喜欢的水果是什么?"; + const summaryOnly = trace("summary_only", 0.9, 0); + const answer = trace("answer", 0.85, 0); + Object.assign(summaryOnly, { + userText: question, + agentText: "好的", + summary: "用户曾经最喜欢的水果是苹果", + }); + Object.assign(answer, { + userText: question, + agentText: "你后来更新了偏好,现在自己最喜欢的水果已经变成芒果。", + }); + + const out = rank({ + tier1: [], + tier2Traces: [summaryOnly, answer], + tier2Episodes: [], + tier3: [], + limit: 2, + config: { + ...cfg, + relativeThresholdFloor: 0, + smartSeed: false, + }, + now: NOW, + }); + + expect(out.ranked.map((item) => String(item.candidate.refId))).toEqual([ + "summary_only", + "answer", + ]); + expect(out.dedupedBeforeMmr).toBe(0); + }); + + it("keeps distinct substantive answers to the same question before MMR", () => { + const question = "我最喜欢的水果是什么?"; + const oldAnswer = trace("old", 0.9, 0); + const newAnswer = trace("new", 0.85, 0); + Object.assign(oldAnswer, { + userText: question, + agentText: "你以前明确说过,当时自己最喜欢的水果一直都是苹果。", + }); + Object.assign(newAnswer, { + userText: question, + agentText: "你后来更新了偏好,现在自己最喜欢的水果已经变成芒果。", + }); + + const out = rank({ + tier1: [], + tier2Traces: [oldAnswer, newAnswer], + tier2Episodes: [], + tier3: [], + limit: 2, + config: { + ...cfg, + relativeThresholdFloor: 0, + smartSeed: false, + }, + now: NOW, + }); + + expect(out.ranked.map((item) => String(item.candidate.refId))).toEqual([ + "old", + "new", + ]); + expect(out.dedupedBeforeMmr).toBe(0); + }); + + it("keeps mixed-dimension candidates without crashing MMR", () => { + const out = rank({ + tier1: [], + tier2Traces: [ + trace("current_dim", 0.9, 0.5, [1, 0, 0]), + trace("legacy_dim", 0.8, 0.5, [1, 0]), + ], + tier2Episodes: [], + tier3: [], + limit: 2, + config: { ...cfg, relativeThresholdFloor: 0 }, + now: NOW, + }); + + expect(out.ranked.map((r) => String(r.candidate.refId))).toEqual([ + "current_dim", + "legacy_dim", + ]); + expect(out.ranked[1]!.scoreDetails?.redundancy).toBe(0); + }); + it("respects `limit`", () => { const ts = [trace("t1", 0.8, 0.2), trace("t2", 0.7, 0.3), trace("t3", 0.6, 0.4)]; const out = rank({ @@ -374,7 +778,7 @@ describe("retrieval/ranker", () => { expect(String(out.ranked[0]!.candidate.refId)).toBe("multi_ch"); }); - it("multi-channel bypass lets low-relevance keyword hits survive threshold", () => { + it("does not let two weak keyword channels bypass the threshold", () => { // Strong candidate pulls topRelevance up; keyword-only single-channel // hit would be guillotined by the relative floor, BUT a multi-channel // hit with the same base should survive via the bypass. @@ -398,11 +802,55 @@ describe("retrieval/ranker", () => { }); const ids = out.ranked.map((r) => String(r.candidate.refId)); expect(ids).toContain("strong"); - expect(ids).toContain("confirmed"); + expect(ids).not.toContain("confirmed"); // The single-channel weak FTS hit should still get cut. expect(ids).not.toContain("fts_only"); }); + it("lets a multi-channel hit with strong semantic evidence bypass the threshold", () => { + const strong = trace("strong", 1, 0); + strong.channels = [{ channel: "vec_summary", rank: 0, score: 1 }]; + const confirmed = trace("confirmed", 0.5, 0); + confirmed.channels = [ + { channel: "vec_summary", rank: 5, score: 0.5 }, + { channel: "pattern", rank: 4, score: 0.2 }, + ]; + + const out = rank({ + tier1: [], + tier2Traces: [strong, confirmed], + tier2Episodes: [], + tier3: [], + limit: 5, + config: { ...cfg, relativeThresholdFloor: 0.6, multiChannelBypass: true }, + now: NOW, + }); + + expect(out.ranked.map((item) => String(item.candidate.refId))).toContain("confirmed"); + }); + + it("uses relevance-focused MMR and ignores V for personal facts", () => { + const first = trace("first", 1, 0, [1, 0]); + const duplicate = trace("duplicate", 0.89, 0, [1, 0]); + const diverseHighV = trace("diverse", 0.6, 1, [0, 1]); + + const out = rank({ + tier1: [], + tier2Traces: [first, duplicate, diverseHighV], + tier2Episodes: [], + tier3: [], + limit: 2, + profile: "personal_fact", + config: { ...cfg, mmrLambda: 0, relativeThresholdFloor: 0 }, + now: NOW, + }); + + expect(out.ranked.map((item) => String(item.candidate.refId))).toEqual([ + "first", + "duplicate", + ]); + }); + it("multiChannelBypass=false restores strict threshold for multi-channel hits", () => { const strong = trace("strong", 0.9, 0.9); strong.channels = [{ channel: "vec_summary", rank: 0, score: 0.9 }]; diff --git a/apps/memos-local-plugin/tests/unit/retrieval/tier1.test.ts b/apps/memos-local-plugin/tests/unit/retrieval/tier1.test.ts index 9c36c1f26..50cf2c4ee 100644 --- a/apps/memos-local-plugin/tests/unit/retrieval/tier1.test.ts +++ b/apps/memos-local-plugin/tests/unit/retrieval/tier1.test.ts @@ -36,6 +36,7 @@ function makeRepo( eta: number; score: number; procedureJson?: unknown; + storedVec?: EmbeddingVector | null; }>, ) { const repo: RetrievalRepos["skills"] = { @@ -61,6 +62,7 @@ function makeRepo( invocationGuide: `run ${r.id}`, procedureJson: r.procedureJson, eta: r.eta, + vec: r.storedVec ?? null, }; }, }; @@ -92,6 +94,23 @@ describe("retrieval/tier1", () => { expect(kept.length).toBe(0); }); + it("hydrates the stored skill vector instead of copying the query vector", async () => { + const storedVec = Float32Array.from([0.8, 0.2, 0]) as EmbeddingVector; + const repo = makeRepo([ + { id: "a", status: "active", eta: 0.9, score: 0.95, storedVec }, + ]); + + const kept = await runTier1( + { repos: { skills: repo }, config: cfg }, + { kind: "embedded", queryVec: qv, rawText: "x" }, + ); + + expect(kept[0]!.vec?.[0]).toBeCloseTo(0.8); + expect(kept[0]!.vec?.[1]).toBeCloseTo(0.2); + expect(kept[0]!.vec?.[2]).toBe(0); + expect(kept[0]!.vec).not.toBe(qv); + }); + it("normalises snake_case skill decision guidance from procedure JSON", async () => { const repo = makeRepo([ { diff --git a/apps/memos-local-plugin/tests/unit/retrieval/tier2.test.ts b/apps/memos-local-plugin/tests/unit/retrieval/tier2.test.ts index 2afacbf56..5d7fae5cb 100644 --- a/apps/memos-local-plugin/tests/unit/retrieval/tier2.test.ts +++ b/apps/memos-local-plugin/tests/unit/retrieval/tier2.test.ts @@ -101,6 +101,182 @@ describe("retrieval/tier2 (with real sqlite)", () => { expect(ids).not.toContain("zeroV"); }); + it("returns the expanded candidate pool instead of truncating before global ranking", async () => { + const out = await runTier2( + { repos: { traces: handle.repos.traces }, config: cfg, now: () => NOW }, + { queryVec: vec([1, 0, 0]), tags: [], includeLowValue: true }, + ); + + expect(out.traces.length).toBeGreaterThan(cfg.tier2TopK); + }); + + it("does not add value priority during Tier-2 pre-ordering", async () => { + handle.repos.traces.insert({ + id: "semanticFirst" as TraceId, + episodeId: "ep1" as EpisodeId, + sessionId: "s1" as SessionId, + ts: NOW as never, + userText: "semantic match", + agentText: "reply", + toolCalls: [], + reflection: null, + value: 0.05 as never, + alpha: 0.5 as never, + rHuman: null, + priority: 0.05 as never, + tags: [], + vecSummary: vec([1, 0, 0]), + vecAction: null, + turnId: 0 as never, + schemaVersion: 1, + }); + handle.repos.traces.insert({ + id: "valuableButWeaker" as TraceId, + episodeId: "ep1" as EpisodeId, + sessionId: "s1" as SessionId, + ts: NOW as never, + userText: "weaker match", + agentText: "reply", + toolCalls: [], + reflection: null, + value: 1 as never, + alpha: 0.5 as never, + rHuman: null, + priority: 1 as never, + tags: [], + vecSummary: vec([0.8, 0.6, 0]), + vecAction: null, + turnId: 0 as never, + schemaVersion: 1, + }); + + const out = await runTier2( + { repos: { traces: handle.repos.traces }, config: cfg, now: () => NOW }, + { queryVec: vec([1, 0, 0]), tags: [] }, + ); + + const ids = out.traces.map((trace) => String(trace.refId)); + expect(ids.indexOf("semanticFirst")).toBeLessThan(ids.indexOf("valuableButWeaker")); + }); + + it("uses summary vectors only for the personal-fact first pass", async () => { + handle.repos.traces.insert({ + id: "actionOnly" as TraceId, + episodeId: "ep1" as EpisodeId, + sessionId: "s1" as SessionId, + ts: NOW as never, + userText: "action-only match", + agentText: "reply", + toolCalls: [], + reflection: null, + value: 0.8 as never, + alpha: 0.5 as never, + rHuman: null, + priority: 0.8 as never, + tags: [], + vecSummary: vec([0, 1, 0]), + vecAction: vec([1, 0, 0]), + turnId: 0 as never, + schemaVersion: 1, + }); + + const out = await runTier2( + { repos: { traces: handle.repos.traces }, config: cfg, now: () => NOW }, + { queryVec: vec([1, 0, 0]), tags: [], profile: "personal_fact" }, + ); + const actionOnly = out.traces.find((trace) => trace.refId === "actionOnly"); + + expect( + actionOnly?.channels?.some((channel) => channel.channel === "vec_action") ?? false, + ).toBe(false); + }); + + it("drops vector hits below the adaptive semantic floor", async () => { + const out = await runTier2( + { repos: { traces: handle.repos.traces }, config: cfg, now: () => NOW }, + { queryVec: vec([1, 0, 0]), tags: [] }, + ); + + expect(out.traces.map((trace) => String(trace.refId))).not.toContain("offTopic"); + }); + + it("hydrates each candidate with its stored vector instead of the query vector", async () => { + const queryVec = vec([1, 0, 0]); + const out = await runTier2( + { repos: { traces: handle.repos.traces }, config: cfg, now: () => NOW }, + { queryVec, tags: ["docker"] }, + ); + const candidate = out.traces.find((trace) => String(trace.refId) === "medV"); + + expect(Array.from(candidate!.vec!)).toEqual( + expect.arrayContaining([expect.closeTo(0.9), expect.closeTo(0.1), 0]), + ); + expect(candidate!.vec).not.toBe(queryVec); + }); + + it("recalls a full identifier through a dedicated channel instead of generic pattern terms", async () => { + const identifier = "project_id_2026_alpha_001"; + handle.repos.traces.insert({ + id: "exactIdentifier" as TraceId, + episodeId: "ep1" as EpisodeId, + sessionId: "s1" as SessionId, + ts: NOW as never, + userText: `项目 ${identifier} 已经完成`, + agentText: "状态已确认", + toolCalls: [], + reflection: null, + value: 0.8 as never, + alpha: 0.5 as never, + rHuman: null, + priority: 0.8 as never, + tags: [], + vecSummary: vec([0, 1, 0]), + vecAction: null, + turnId: 0 as never, + schemaVersion: 1, + }); + handle.repos.traces.insert({ + id: "genericPattern" as TraceId, + episodeId: "ep1" as EpisodeId, + sessionId: "s1" as SessionId, + ts: NOW as never, + userText: "请查询之前的普通项目记录", + agentText: "状态已确认", + toolCalls: [], + reflection: null, + value: 0.8 as never, + alpha: 0.5 as never, + rHuman: null, + priority: 0.8 as never, + tags: [], + vecSummary: vec([0, 1, 0]), + vecAction: null, + turnId: 0 as never, + schemaVersion: 1, + }); + + const out = await runTier2( + { repos: { traces: handle.repos.traces }, config: cfg, now: () => NOW }, + { + queryVec: null, + tags: [], + patternTerms: ["查询"], + exactIdentifiers: [identifier], + }, + ); + + const exact = out.traces.find((candidate) => + String(candidate.refId) === "exactIdentifier" + ); + expect(exact?.channels).toContainEqual( + expect.objectContaining({ channel: "exact_identifier" }), + ); + expect( + out.traces.find((candidate) => String(candidate.refId) === "genericPattern") + ?.channels, + ).toContainEqual(expect.objectContaining({ channel: "pattern" })); + }); + it("tag filter narrows candidate set", async () => { const out = await runTier2( { repos: { traces: handle.repos.traces }, config: cfg, now: () => NOW }, diff --git a/apps/memos-local-plugin/tests/unit/retrieval/tier3.test.ts b/apps/memos-local-plugin/tests/unit/retrieval/tier3.test.ts index 549d94c02..41acce2b1 100644 --- a/apps/memos-local-plugin/tests/unit/retrieval/tier3.test.ts +++ b/apps/memos-local-plugin/tests/unit/retrieval/tier3.test.ts @@ -70,14 +70,17 @@ describe("retrieval/tier3 (with real sqlite)", () => { afterEach(() => handle.cleanup()); it("returns the closest world-model by cosine", async () => { + const queryVec = vec([0.8, 0.6, 0]); const out = await runTier3( { repos: { worldModel: handle.repos.worldModel }, config: cfg }, - { queryVec: vec([1, 0, 0]) }, + { queryVec }, ); expect(out.length).toBeGreaterThanOrEqual(1); expect(String(out[0]!.refId)).toBe("wm_docker"); expect(out[0]!.title).toBe("docker-compose"); expect(out[0]!.body).toContain("containers"); + expect(Array.from(out[0]!.vec!)).toEqual([1, 0, 0]); + expect(out[0]!.vec).not.toBe(queryVec); }); it("returns a candidate pool sized by tier3TopK · candidatePoolFactor", async () => { diff --git a/apps/memos-local-plugin/tests/unit/storage/fts-keyword.test.ts b/apps/memos-local-plugin/tests/unit/storage/fts-keyword.test.ts index 0024f797c..c124401a9 100644 --- a/apps/memos-local-plugin/tests/unit/storage/fts-keyword.test.ts +++ b/apps/memos-local-plugin/tests/unit/storage/fts-keyword.test.ts @@ -26,6 +26,7 @@ describe("storage/keyword channels — traces", () => { agentText?: string; summary?: string; tags?: string[]; + ts?: number; }, ): void { // Use the trace id to disambiguate session+episode so multiple @@ -52,7 +53,7 @@ describe("storage/keyword channels — traces", () => { id: opts.id as never, episodeId: eid as never, sessionId: sid as never, - ts: Date.now() as never, + ts: (opts.ts ?? Date.now()) as never, userText: opts.userText, agentText: opts.agentText ?? "", toolCalls: [], @@ -130,6 +131,30 @@ describe("storage/keyword channels — traces", () => { } }); + it("ranks pattern hits by term coverage instead of update time", () => { + const handle = makeTmpDb(); + try { + seedTrace(handle.repos, { + id: "t_full_old", + userText: "我喜欢的歌手是陈奕迅", + ts: 100, + }); + seedTrace(handle.repos, { + id: "t_partial_new", + userText: "我喜欢散步", + ts: 200, + }); + + const hits = handle.repos.traces.searchByPattern(["喜欢", "歌手"], 10); + + expect(hits.map((hit) => hit.id)).toEqual(["t_full_old", "t_partial_new"]); + expect(hits[0]!.score).toBe(1); + expect(hits[1]!.score).toBe(0.5); + } finally { + handle.cleanup(); + } + }); + it("trigger keeps FTS in sync on INSERT / UPDATE / DELETE", () => { const handle = makeTmpDb(); try { diff --git a/apps/memos-local-plugin/tests/unit/storage/keyword.test.ts b/apps/memos-local-plugin/tests/unit/storage/keyword.test.ts index 62bf20c2e..86dab1605 100644 --- a/apps/memos-local-plugin/tests/unit/storage/keyword.test.ts +++ b/apps/memos-local-plugin/tests/unit/storage/keyword.test.ts @@ -71,9 +71,27 @@ describe("storage/keyword.extractPatternTerms", () => { it("emits CJK bigrams over each CJK run", () => { const out = extractPatternTerms("帮我部署"); - expect(out).toContain("帮我"); - expect(out).toContain("我部"); expect(out).toContain("部署"); + expect(out).not.toContain("帮我"); + expect(out).not.toContain("我部"); + }); + + it("removes conversational CJK bigrams from long personal-fact queries", () => { + const out = extractPatternTerms("你还记得我喜欢什么吗"); + + expect(out).toContain("喜欢"); + expect(out).not.toContain("你还"); + expect(out).not.toContain("还记"); + expect(out).not.toContain("得我"); + expect(out).not.toContain("什么"); + expect(out).not.toContain("么吗"); + }); + + it("keeps content bigrams while removing grammatical bridges", () => { + const out = extractPatternTerms("我爱吃的水果呢"); + + expect(out).toEqual(expect.arrayContaining(["爱吃", "水果"])); + expect(out).not.toEqual(expect.arrayContaining(["我爱", "吃的", "的水", "果呢"])); }); it("keeps 2-char CJK runs as a single bigram", () => { diff --git a/apps/memos-local-plugin/viewer/src/views/LogsView.tsx b/apps/memos-local-plugin/viewer/src/views/LogsView.tsx index 659bdf3cf..93cfd6460 100644 --- a/apps/memos-local-plugin/viewer/src/views/LogsView.tsx +++ b/apps/memos-local-plugin/viewer/src/views/LogsView.tsx @@ -596,6 +596,9 @@ interface RetrievalStatsPayload { raw?: number; ranked?: number; droppedByThreshold?: number; + dedupedBeforeMmr?: number; + dedupedAfterThreshold?: number; + droppedByKeywordConfirmation?: number; thresholdFloor?: number; topRelevance?: number; llmFilter?: { @@ -607,6 +610,7 @@ interface RetrievalStatsPayload { channelHits?: Record; queryTokens?: number; queryTags?: string[]; + exactIdentifierCount?: number; localReturned?: number; hubReturned?: number; hubKept?: number; @@ -725,6 +729,7 @@ function RetrievalFunnel({ stats }: { stats: RetrievalStatsPayload }) { const localFilterDeferred = outcome === "deferred_to_final"; const finalLlmRan = finalFilter?.outcome === "llm_kept_all" || finalFilter?.outcome === "llm_filtered" || + finalFilter?.outcome === "llm_filtered_empty" || finalFilter?.outcome === "llm_filtered_refilled" || finalFilter?.outcome === "llm_failed_safe_cutoff"; const fmtNum = (n: number | undefined, digits = 3) => @@ -750,9 +755,33 @@ function RetrievalFunnel({ stats }: { stats: RetrievalStatsPayload }) { )} raw {raw} ranked {ranked} + {typeof stats.exactIdentifierCount === "number" && + stats.exactIdentifierCount > 0 && ( + + exact identifiers {stats.exactIdentifierCount} + + )} {dropped > 0 && ( dropped≥floor {dropped} )} + {typeof stats.dedupedBeforeMmr === "number" && + stats.dedupedBeforeMmr > 0 && ( + + deduped before MMR {stats.dedupedBeforeMmr} + + )} + {typeof stats.dedupedAfterThreshold === "number" && + stats.dedupedAfterThreshold > 0 && ( + + of which after threshold {stats.dedupedAfterThreshold} + + )} + {typeof stats.droppedByKeywordConfirmation === "number" && + stats.droppedByKeywordConfirmation > 0 && ( + + identifier rejected {stats.droppedByKeywordConfirmation} + + )} {typeof kept === "number" && ( {localFilterDeferred ? "local candidates" : "local llm kept"} {kept} From 3daaaadcc44df61f31a9bca00c9f093b5ceea5ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=81=E5=9C=A8=E5=90=B5=E7=9D=80=E5=90=83=E7=B3=96?= Date: Fri, 31 Jul 2026 18:04:31 +0800 Subject: [PATCH 10/18] fix(plugin): use clean OpenClaw user input --- .../adapters/openclaw/bridge.ts | 155 ++++++++++++++++-- .../adapters/openclaw/index.ts | 24 ++- .../adapters/openclaw/openclaw-api.ts | 32 ++++ .../unit/adapters/openclaw-bridge.test.ts | 133 +++++++++++++++ .../unit/adapters/openclaw-runtime.test.ts | 58 +++++++ 5 files changed, 383 insertions(+), 19 deletions(-) diff --git a/apps/memos-local-plugin/adapters/openclaw/bridge.ts b/apps/memos-local-plugin/adapters/openclaw/bridge.ts index f849fe5f1..210fdb6f5 100644 --- a/apps/memos-local-plugin/adapters/openclaw/bridge.ts +++ b/apps/memos-local-plugin/adapters/openclaw/bridge.ts @@ -38,7 +38,9 @@ import type { BeforePromptBuildResult, BeforeToolCallEvent, HostLogger, + MessageReceivedEvent, PluginHookAgentContext, + PluginHookMessageContext, PluginHookSessionContext, PluginHookSubagentContext, PluginHookToolContext, @@ -512,8 +514,21 @@ function stripOpenClawUserEnvelope(raw: string): string { // 4. Leading timestamp like "[Thu 2026-03-05 15:23 GMT+8] " text = text.replace(/^\[(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun)\s+[^\]]+\]\s*/, ""); - // 5. Inline envelope tags OpenClaw leaves behind. - text = text.replace(/\[message_id:\s*[a-f0-9-]+\]/gi, ""); + // 5. Legacy channel decoration. Modern OpenClaw paths use the clean + // `message_received.content` value instead (see + // `OpenClawInboundTextStore` below). These conservative rules exist + // only for old hosts, internal runs, and cache misses: + // [message_id: om_x...] + // ou_x...: actual Feishu message + // + // Match at the beginning only. A user may legitimately discuss a + // string such as `[message_id: example]` in their message; stripping + // arbitrary occurrences would alter real input. + text = text.replace( + /^(?:\s*\[message_id:\s*[^\]\r\n]+\]\s*(?:\r?\n)?)+/i, + "", + ); + text = text.replace(/^\s*ou_[a-z0-9_-]+:\s*/i, ""); text = text.replace(/\[\[reply_to_current\]\]/gi, ""); // 6. Line-level OpenClaw side-channel injections. OpenClaw appends @@ -847,10 +862,96 @@ function capContextBlock(block: string): { block: string; truncated: boolean } { // ─── Bridge factory ──────────────────────────────────────────────────────── +const DEFAULT_INBOUND_TEXT_TTL_MS = 2 * 60 * 60 * 1_000; +const DEFAULT_INBOUND_TEXT_MAX_ENTRIES = 1_024; + +interface InboundTextEntry { + content: string; + rememberedAt: number; +} + +/** + * Short-lived correlation store between OpenClaw's `message_received` + * hook and its later agent hooks. + * + * `runId` is deliberately the only key. OpenClaw documents it as stable + * across one inbound turn and distinct between concurrent turns. Falling + * back to sessionKey/messageId here would risk pairing the wrong user text + * when two messages overlap; older hosts without runId instead use the + * conservative envelope sanitizer. + */ +export interface OpenClawInboundTextStore { + remember: ( + event: MessageReceivedEvent, + ctx: PluginHookMessageContext, + ) => void; + get: (runId: string | undefined) => string | undefined; + delete: (runId: string | undefined) => void; + size: () => number; +} + +export interface OpenClawInboundTextStoreOptions { + now?: () => number; + ttlMs?: number; + maxEntries?: number; +} + +export function createOpenClawInboundTextStore( + options: OpenClawInboundTextStoreOptions = {}, +): OpenClawInboundTextStore { + const now = options.now ?? (() => Date.now()); + const ttlMs = Math.max(1, options.ttlMs ?? DEFAULT_INBOUND_TEXT_TTL_MS); + const maxEntries = Math.max( + 1, + Math.floor(options.maxEntries ?? DEFAULT_INBOUND_TEXT_MAX_ENTRIES), + ); + const entries = new Map(); + + function prune(timestamp: number): void { + for (const [runId, entry] of entries) { + if (timestamp - entry.rememberedAt > ttlMs) entries.delete(runId); + } + } + + return { + remember(event, ctx) { + const runId = firstString(event.runId, ctx.runId); + const content = typeof event.content === "string" ? event.content.trim() : ""; + if (!runId || !content) return; + + const timestamp = now(); + prune(timestamp); + // Refresh insertion order so capacity eviction removes the oldest + // turn, not a recently retried message_received delivery. + entries.delete(runId); + entries.set(runId, { content, rememberedAt: timestamp }); + while (entries.size > maxEntries) { + const oldest = entries.keys().next().value as string | undefined; + if (!oldest) break; + entries.delete(oldest); + } + }, + get(runId) { + prune(now()); + if (!runId) return undefined; + return entries.get(runId)?.content; + }, + delete(runId) { + if (runId) entries.delete(runId); + }, + size() { + prune(now()); + return entries.size; + }, + }; +} + export interface BridgeOptions { agent: AgentKind; core: MemoryCore; log: HostLogger; + /** Clean inbound text captured from OpenClaw's `message_received`. */ + inboundUserText?: OpenClawInboundTextStore; /** Override the wall-clock source (tests). */ now?: () => number; } @@ -1165,32 +1266,31 @@ export function createOpenClawBridge(opts: BridgeOptions): BridgeHandle { }); return; } - // Strip OpenClaw's envelope before the text leaks anywhere - // downstream (retrieval query, stored episode.initialTurn, capture - // userText). Without this, every captured memory would carry the - // "[Thu … GMT+8]" prefix and the "Sender (untrusted metadata)" - // block — exactly the bug the user hit. const rawPrompt = (event.prompt ?? "").trim(); - // V7 parity with legacy adapter: don't create episodes for - // OpenClaw boot checks / bootstrap preludes / sentinel replies. - // They're not user input, and retrieving against them wastes a - // tier-2 query and pollutes the viewer. - if (isOpenClawBootstrapMessage(rawPrompt)) { + // Primary path: use OpenClaw's channel-normalized official body. + // Only old hosts, internal runs, or a missed runId correlation + // inspect the model-facing prompt and its channel wrappers. + const prompt = + opts.inboundUserText?.get(ctx.runId) ?? + stripOpenClawUserEnvelope(rawPrompt); + if (!prompt) return; + // Apply runtime-message filters to the selected user text, not the + // wrapped prompt. A model-facing prompt can contain stale internal + // event lines ahead of a perfectly valid official inbound body. + if (isOpenClawBootstrapMessage(prompt)) { opts.log.debug("memos.onTurnStart.skipped_bootstrap", { sessionKey: ctx.sessionKey, - head: rawPrompt.slice(0, 60), + head: prompt.slice(0, 60), }); return; } - if (isOpenClawSubagentAnnouncementPrompt(rawPrompt)) { + if (isOpenClawSubagentAnnouncementPrompt(prompt)) { opts.log.debug("memos.onTurnStart.skipped_subagent_announcement", { sessionKey: ctx.sessionKey, agentId: ctx.agentId, }); return; } - const prompt = stripOpenClawUserEnvelope(rawPrompt); - if (!prompt) return; const namespace = namespaceFromAgentCtx(ctx); const sessionId = await ensureSession(ctx.agentId, ctx.sessionKey, namespace); @@ -1278,6 +1378,7 @@ export function createOpenClawBridge(opts: BridgeOptions): BridgeHandle { if (isEphemeralSessionKey(ctx.sessionKey)) { // Mirror `handleBeforePrompt` — slug-generator & co. don't get a // trace / episode, so there's nothing to persist here either. + opts.inboundUserText?.delete(ctx.runId); return; } const namespace = namespaceFromAgentCtx(ctx); @@ -1320,7 +1421,23 @@ export function createOpenClawBridge(opts: BridgeOptions): BridgeHandle { messageCursor.set(sessionId, allMessages.length); const flat = flattenMessages(novel); - const turn = extractTurn(flat, now()); + const officialUserText = opts.inboundUserText?.get(ctx.runId); + let turn = extractTurn(flat, now()); + if (officialUserText) { + if (turn) { + // Do not sanitize official content. It is already the + // channel-normalized BodyForCommands/RawBody value and may + // legitimately contain text resembling an envelope marker. + turn.userText = officialUserText; + } else { + // Defensive compatibility for hosts that omit the user entry + // from agent_end.messages but still publish message_received. + turn = extractTurn( + [{ role: "user", content: officialUserText }, ...flat], + now(), + ); + } + } if (!turn || !turn.userText) { // Elevated to WARN so unexpected skips show up in the gateway // log. `flat.length` / `novel.length` help diagnose whether the @@ -1429,6 +1546,10 @@ export function createOpenClawBridge(opts: BridgeOptions): BridgeHandle { err: err instanceof Error ? err.message : String(err), stack: err instanceof Error ? err.stack : undefined, }); + } finally { + // before_prompt_build may retry within one run, so the entry is + // only consumed after the terminal agent_end path. + opts.inboundUserText?.delete(ctx.runId); } } diff --git a/apps/memos-local-plugin/adapters/openclaw/index.ts b/apps/memos-local-plugin/adapters/openclaw/index.ts index 018880651..702df318a 100644 --- a/apps/memos-local-plugin/adapters/openclaw/index.ts +++ b/apps/memos-local-plugin/adapters/openclaw/index.ts @@ -28,7 +28,12 @@ import { existsSync, readFileSync } from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; -import { createOpenClawBridge, type BridgeHandle } from "./bridge.js"; +import { + createOpenClawBridge, + createOpenClawInboundTextStore, + type BridgeHandle, + type OpenClawInboundTextStore, +} from "./bridge.js"; import { acquireOpenClawRuntimeLock, DuplicateOpenClawRuntimeError, @@ -135,6 +140,7 @@ const OPENCLAW_VIEWER_PORT = 18799; async function createRuntime( api: OpenClawPluginApi, runtimeLock: OpenClawRuntimeLockHandle, + inboundUserText: OpenClawInboundTextStore, ): Promise { const log = rootLogger.child({ channel: "adapters.openclaw" }); log.info("plugin.bootstrap", { version: PLUGIN_VERSION }); @@ -187,6 +193,7 @@ async function createRuntime( agent: "openclaw", core, log: api.logger, + inboundUserText, }); // OpenClaw's viewer port is fixed at :18799 (hermes uses :18800). @@ -328,6 +335,11 @@ function register(api: OpenClawPluginApi): void { throw err; } + // OpenClaw publishes its clean, command-facing inbound body before + // prompt construction. Keep this store independent of core bootstrap + // so early messages are not lost while SQLite/providers initialize. + const inboundUserText = createOpenClawInboundTextStore(); + // 1. Memory capability (prompt prelude) — register synchronously so the // host immediately knows who owns the memory slot, even if bootstrap // fails later. @@ -385,7 +397,7 @@ function register(api: OpenClawPluginApi): void { // tools register a shell now and wait for runtime inside execute(). let runtime: PluginRuntime | null = null; let bootstrapError: Error | null = null; - const bootstrapPromise = createRuntime(api, runtimeLock) + const bootstrapPromise = createRuntime(api, runtimeLock, inboundUserText) .then((r) => { runtime = r; api.logger.info("memos-local: plugin ready"); @@ -467,6 +479,14 @@ function register(api: OpenClawPluginApi): void { // `prependContext` for OpenClaw to inject — it is a value-returning // hook, not a void hook, and OpenClaw is willing to await its result // (the timeout is laxer than `agent_end`'s 30 s budget). + api.on("message_received", (event, ctx) => { + // Synchronous and intentionally independent of runtime readiness. + // `event.content` is OpenClaw's BodyForCommands/RawBody value, while + // before_prompt_build.prompt and agent_end.messages are model-facing + // and may contain IM-specific sender/message-id decoration. + inboundUserText.remember(event, ctx); + }); + api.on("before_prompt_build", async (event, ctx) => { const r = await ensureRuntime(); if (!r) return; diff --git a/apps/memos-local-plugin/adapters/openclaw/openclaw-api.ts b/apps/memos-local-plugin/adapters/openclaw/openclaw-api.ts index 06bf10d6d..411a1e917 100644 --- a/apps/memos-local-plugin/adapters/openclaw/openclaw-api.ts +++ b/apps/memos-local-plugin/adapters/openclaw/openclaw-api.ts @@ -92,6 +92,7 @@ export interface OpenClawPluginToolOptions { /** Hook names we actually subscribe to. */ export type OpenClawHookName = + | "message_received" | "before_prompt_build" | "agent_end" | "before_tool_call" @@ -115,6 +116,16 @@ export interface PluginHookAgentContext { channelId?: string; } +export interface PluginHookMessageContext { + channelId: string; + accountId?: string; + conversationId?: string; + sessionKey?: string; + runId?: string; + messageId?: string; + senderId?: string; +} + export interface PluginHookSessionContext { agentId?: string; sessionId: string; @@ -141,6 +152,23 @@ export interface BeforePromptBuildEvent { messages: unknown[]; // opaque AgentMessage[] } +/** + * OpenClaw's channel-normalized inbound event. `content` comes from + * `BodyForCommands ?? RawBody ?? Body`, so unlike `BodyForAgent` it does + * not contain channel-specific sender/message-id prompt decoration. + */ +export interface MessageReceivedEvent { + from: string; + content: string; + timestamp?: number; + threadId?: string | number; + messageId?: string; + senderId?: string; + sessionKey?: string; + runId?: string; + metadata?: Record; +} + /** The only return shape OpenClaw reads back from `before_prompt_build`. */ export interface BeforePromptBuildResult { systemPrompt?: string; @@ -228,6 +256,10 @@ export interface SubagentEndedEvent { /** Handler map — each hook has its own event + ctx shape. */ export interface OpenClawHookHandlerMap { + message_received: ( + event: MessageReceivedEvent, + ctx: PluginHookMessageContext, + ) => void | Promise; before_prompt_build: ( event: BeforePromptBuildEvent, ctx: PluginHookAgentContext, diff --git a/apps/memos-local-plugin/tests/unit/adapters/openclaw-bridge.test.ts b/apps/memos-local-plugin/tests/unit/adapters/openclaw-bridge.test.ts index 335a41756..839a7b741 100644 --- a/apps/memos-local-plugin/tests/unit/adapters/openclaw-bridge.test.ts +++ b/apps/memos-local-plugin/tests/unit/adapters/openclaw-bridge.test.ts @@ -26,6 +26,7 @@ import { fakeEmbedder } from "../../helpers/fake-embedder.js"; import { bridgeSessionId, + createOpenClawInboundTextStore, createOpenClawBridge, extractTurn, flattenMessages, @@ -788,6 +789,138 @@ describe("bridgeSessionId", () => { }); }); +describe("OpenClaw official inbound user text", () => { + it("keeps concurrent runs isolated and expires stale entries", () => { + let timestamp = 1_000; + const store = createOpenClawInboundTextStore({ + now: () => timestamp, + ttlMs: 100, + maxEntries: 2, + }); + + store.remember( + { from: "ou_a", content: "第一个正文", timestamp, runId: "run-a" }, + { channelId: "feishu", runId: "run-a" }, + ); + store.remember( + { from: "ou_b", content: "第二个正文", timestamp, runId: "run-b" }, + { channelId: "feishu", runId: "run-b" }, + ); + + expect(store.get("run-a")).toBe("第一个正文"); + expect(store.get("run-b")).toBe("第二个正文"); + + store.remember( + { from: "ou_c", content: "第三个正文", timestamp, runId: "run-c" }, + { channelId: "feishu", runId: "run-c" }, + ); + expect(store.get("run-a")).toBeUndefined(); + expect(store.get("run-b")).toBe("第二个正文"); + expect(store.get("run-c")).toBe("第三个正文"); + + timestamp += 101; + expect(store.get("run-b")).toBeUndefined(); + expect(store.size()).toBe(0); + }); + + it("uses message_received content for retrieval without sanitizing user-authored text", async () => { + const mc = buildCore(); + await mc.init(); + const onTurnStart = vi.spyOn(mc, "onTurnStart"); + const inboundUserText = createOpenClawInboundTextStore(); + const ctx = hookCtx({ sessionKey: "s-official-search", runId: "run-official-search" }); + const cleanText = "[message_id: abc-def] 这是用户主动输入的正文"; + inboundUserText.remember( + { + from: "ou_14ad16b11b12b0322be96b71c26713b1", + content: cleanText, + timestamp: Date.now(), + runId: ctx.runId, + }, + { channelId: "feishu", runId: ctx.runId, sessionKey: ctx.sessionKey }, + ); + + const bridge = createOpenClawBridge({ + agent: "openclaw", + core: mc, + log: silentLogger(), + inboundUserText, + }); + await bridge.handleBeforePrompt( + { + prompt: + "An async command completion event was triggered, but user delivery is disabled.\n" + + "[message_id: om_x100b69f11b8858acde918bb5c01ea62]\n" + + "ou_14ad16b11b12b0322be96b71c26713b1: 被包装的模型输入", + messages: [], + }, + ctx, + ); + + expect(onTurnStart).toHaveBeenCalledWith( + expect.objectContaining({ userText: cleanText }), + ); + }); + + it("uses message_received content when agent_end persists a cache-only turn", async () => { + const mc = buildCore(); + await mc.init(); + const inboundUserText = createOpenClawInboundTextStore(); + const ctx = hookCtx({ sessionKey: "s-official-add", runId: "run-official-add" }); + const cleanText = "你还记得我喜欢什么歌吗"; + inboundUserText.remember( + { + from: "ou_14ad16b11b12b0322be96b71c26713b1", + content: cleanText, + timestamp: Date.now(), + runId: ctx.runId, + }, + { channelId: "feishu", runId: ctx.runId, sessionKey: ctx.sessionKey }, + ); + const bridge = createOpenClawBridge({ + agent: "openclaw", + core: mc, + log: silentLogger(), + inboundUserText, + }); + + await bridge.handleAgentEnd( + { + success: true, + messages: [ + { + role: "user", + content: + "[message_id: om_x100b69f11b8858acde918bb5c01ea62]\n" + + "ou_14ad16b11b12b0322be96b71c26713b1: 被包装的模型输入", + }, + { role: "assistant", content: "当然记得。" }, + ], + }, + ctx, + ); + await (pipeline as PipelineHandle).flush(); + + const traces = await mc.listTraces({ groupByTurn: true }); + expect(traces).toHaveLength(1); + expect(traces[0]!.userText).toBe(cleanText); + expect(inboundUserText.get(ctx.runId!)).toBeUndefined(); + }); + + it("falls back to conservative Feishu envelope cleanup on a cache miss", () => { + const flat = flattenMessages([ + { + role: "user", + content: + "[message_id: om_x100b69f11b8858acde918bb5c01ea62]\n" + + "ou_14ad16b11b12b0322be96b71c26713b1: 你还记得我喜欢什么歌吗", + }, + ]); + + expect(extractTurn(flat, 0)?.userText).toBe("你还记得我喜欢什么歌吗"); + }); +}); + describe("ephemeral-session filter", () => { // Regression: OpenClaw's slug-generator sub-agent shares the plugin // host via `sessionKey: "temp:slug-generator"`. Before this filter, diff --git a/apps/memos-local-plugin/tests/unit/adapters/openclaw-runtime.test.ts b/apps/memos-local-plugin/tests/unit/adapters/openclaw-runtime.test.ts index 401acb9aa..8a546c59b 100644 --- a/apps/memos-local-plugin/tests/unit/adapters/openclaw-runtime.test.ts +++ b/apps/memos-local-plugin/tests/unit/adapters/openclaw-runtime.test.ts @@ -257,6 +257,63 @@ function buildBridgeStub() { } describe("OpenClaw hook listener contract (issue #1815)", () => { + it("captures message_received content synchronously before runtime bootstrap completes", async () => { + const home = useTempMemosHome(); + const core = makeCore(); + const bootDeferred = deferred<{ + core: ReturnType; + config: typeof DEFAULT_CONFIG; + home: ResolvedHome; + }>(); + const bootstrapMemoryCoreFull = vi.fn(() => bootDeferred.promise); + const startHttpServer = vi.fn(async () => ({ + url: "http://127.0.0.1:18799", + port: 18799, + closed: false, + close: vi.fn(async () => {}), + })); + const bridge = buildBridgeStub(); + const createOpenClawBridge = vi.fn(() => bridge); + const plugin = await loadPluginWithMocks( + bootstrapMemoryCoreFull, + startHttpServer, + createOpenClawBridge, + ); + const api = makeApi(); + plugin.register(api); + + const handler = api.hooks.get("message_received") as + | OpenClawHookHandlerMap["message_received"] + | undefined; + expect(handler).toBeDefined(); + expect(handler!.constructor.name).not.toBe("AsyncFunction"); + expect( + handler!( + { + from: "ou_sender", + content: "OpenClaw 官方干净正文", + runId: "run-clean", + }, + { + channelId: "feishu", + runId: "run-clean", + sessionKey: "agent:main:main", + }, + ), + ).toBeUndefined(); + + bootDeferred.resolve({ core, config: DEFAULT_CONFIG, home }); + await api.services[0]!.start?.(); + + const bridgeOptions = createOpenClawBridge.mock.calls[0]![0] as { + inboundUserText?: { get: (runId: string) => string | undefined }; + }; + expect(bridgeOptions.inboundUserText?.get("run-clean")).toBe( + "OpenClaw 官方干净正文", + ); + await api.services[0]!.stop?.(); + }); + it("registers tool_result_persist as a SYNC listener that returns the bridge result directly (not a Promise)", async () => { const bridge = buildBridgeStub(); const plugin = (await buildPluginWithFakeBridge({ bridge })) as { @@ -455,6 +512,7 @@ describe("OpenClaw hook listener contract (issue #1815)", () => { // before_prompt_build is value-returning and the only listener // allowed to be async (OpenClaw awaits its prependContext). const voidHooks: OpenClawHookName[] = [ + "message_received", "agent_end", "before_tool_call", "after_tool_call", From b1436bf22f516176667e7a53869153dbcf641005 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=81=E5=9C=A8=E5=90=B5=E7=9D=80=E5=90=83=E7=B3=96?= Date: Fri, 31 Jul 2026 18:04:56 +0800 Subject: [PATCH 11/18] fix(plugin): ignore OpenClaw internal wakeups --- .../adapters/openclaw/bridge.ts | 6 ++ .../unit/adapters/openclaw-bridge.test.ts | 60 +++++++++++++++++++ 2 files changed, 66 insertions(+) diff --git a/apps/memos-local-plugin/adapters/openclaw/bridge.ts b/apps/memos-local-plugin/adapters/openclaw/bridge.ts index 210fdb6f5..e3e85515f 100644 --- a/apps/memos-local-plugin/adapters/openclaw/bridge.ts +++ b/apps/memos-local-plugin/adapters/openclaw/bridge.ts @@ -382,6 +382,12 @@ const OPENCLAW_BOOT_SIGNATURES: readonly string[] = [ // Source signatures live in OpenClaw `infra/heartbeat-events-filter.ts` // and `auto-reply/reply/session-reset-prompt.ts`. "An async command you ran earlier has completed", + // OpenClaw 2026.7+ uses this shared prefix for both "no command + // output" and "user delivery is disabled" heartbeat wakeups. + "An async command completion event was triggered", + // Session persistence may retain this compact marker even when + // before_prompt_build receives the expanded prompt above. + "[OpenClaw heartbeat poll]", "A scheduled reminder has been triggered", "A scheduled cron event was triggered", "Run the following periodic tasks", diff --git a/apps/memos-local-plugin/tests/unit/adapters/openclaw-bridge.test.ts b/apps/memos-local-plugin/tests/unit/adapters/openclaw-bridge.test.ts index 839a7b741..1adb51081 100644 --- a/apps/memos-local-plugin/tests/unit/adapters/openclaw-bridge.test.ts +++ b/apps/memos-local-plugin/tests/unit/adapters/openclaw-bridge.test.ts @@ -30,6 +30,7 @@ import { createOpenClawBridge, extractTurn, flattenMessages, + isOpenClawBootstrapMessage, renderContextBlock, } from "../../../adapters/openclaw/bridge.js"; import { registerOpenClawTools } from "../../../adapters/openclaw/tools.js"; @@ -635,6 +636,22 @@ describe("extractTurn", () => { * If you change one, change the other in lockstep. */ describe("OpenClaw side-channel user injections", () => { + const currentAsyncCompletionPrompts = [ + "An async command completion event was triggered, but no command output was found. " + + "Reply HEARTBEAT_OK only. Do not mention, summarize, or reuse output from any earlier run.", + "An async command completion event was triggered, but user delivery is disabled for this run. " + + "Handle the result internally and reply HEARTBEAT_OK only. " + + "Do not mention, summarize, or reuse command output.\n" + + "Reference UTC: 2026-07-31 08:05 UTC", + ]; + + it("recognizes current OpenClaw async-completion prompts as bootstrap", () => { + for (const prompt of currentAsyncCompletionPrompts) { + expect(isOpenClawBootstrapMessage(prompt)).toBe(true); + } + expect(isOpenClawBootstrapMessage("[OpenClaw heartbeat poll]")).toBe(true); + }); + it("drops the async-exec-completion wakeup prompt as bootstrap (no captured turn)", () => { const flat = flattenMessages([ { @@ -735,6 +752,49 @@ describe("OpenClaw side-channel user injections", () => { // `if (!turn.userText) return;` then short-circuits. expect(turn?.userText).toBe(""); }); + + it("does not search, route an episode, or capture current async-completion prompts", async () => { + const mc = buildCore(); + await mc.init(); + const bridge = createOpenClawBridge({ + agent: "openclaw", + core: mc, + log: silentLogger(), + }); + + for (const [index, prompt] of currentAsyncCompletionPrompts.entries()) { + const ctx = hookCtx({ + sessionKey: `agent:main:heartbeat:${index}`, + runId: `heartbeat-run-${index}`, + }); + const retrieval = await bridge.handleBeforePrompt( + { prompt, messages: [] }, + ctx, + ); + expect(retrieval).toBeUndefined(); + + await bridge.handleAgentEnd( + { + success: true, + messages: [ + { role: "user", content: prompt }, + { role: "assistant", content: "HEARTBEAT_OK" }, + ], + }, + ctx, + ); + } + await (pipeline as PipelineHandle).flush(); + + const searchLogs = await mc.listApiLogs({ + toolName: "memos_search", + limit: 10, + }); + expect(searchLogs.logs).toHaveLength(0); + expect(searchLogs.total).toBe(0); + expect(await mc.listEpisodeRows({ limit: 10 })).toHaveLength(0); + expect(await mc.listTraces({ limit: 10 })).toHaveLength(0); + }); }); describe("renderContextBlock", () => { From 3625ed86a37d797824aec8619e308ffc1c80268a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=81=E5=9C=A8=E5=90=B5=E7=9D=80=E5=90=83=E7=B3=96?= Date: Fri, 31 Jul 2026 18:05:24 +0800 Subject: [PATCH 12/18] fix(plugin): recall compacted same-session history --- .../hermes/memos_provider/__init__.py | 31 ++++++- .../adapters/openclaw/bridge.ts | 40 +++++++- .../core/pipeline/retrieval-repos.ts | 47 +++++++++- .../core/retrieval/retrieve.ts | 52 ++++++++++- .../core/retrieval/tier2-trace.ts | 49 +++++++++- .../python/test_hermes_provider_pipeline.py | 25 +++++ .../unit/adapters/openclaw-bridge.test.ts | 36 ++++++++ .../unit/pipeline/retrieval-repos.test.ts | 92 +++++++++++++++++++ .../tests/unit/retrieval/integration.test.ts | 52 +++++++++++ .../tests/unit/retrieval/tier2.test.ts | 60 ++++++++++-- 10 files changed, 456 insertions(+), 28 deletions(-) create mode 100644 apps/memos-local-plugin/tests/unit/pipeline/retrieval-repos.test.ts diff --git a/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py b/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py index 027f21ba0..6eeaa000e 100644 --- a/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py +++ b/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py @@ -381,6 +381,12 @@ def __init__(self) -> None: self._state_lock = threading.Lock() self._last_injected_context: str = "" self._active_turn_key: str = "" + # Hermes does not pass the live conversation history to `prefetch`. + # Once `on_pre_compress` fires, this timestamp separates history + # that has fallen out of the raw model window from newer turns that + # are still visible. Before the first compression it stays unset so + # the core preserves the legacy whole-session no-repeat behaviour. + self._visible_context_start_ts: int | None = None # Tool calls accumulated via the Hermes `post_tool_call` plugin # hook — flushed alongside user/assistant text in `sync_turn`. self._tool_calls: list[dict[str, Any]] = [] @@ -452,6 +458,7 @@ def initialize(self, session_id: str, **kwargs: Any) -> None: # type: ignore[ov with self._state_lock: self._last_injected_context = "" self._active_turn_key = "" + self._visible_context_start_ts = None self._hermes_home = str(kwargs.get("hermes_home") or "") self._platform = str(kwargs.get("platform") or "cli") self._agent_identity = str(kwargs.get("agent_identity") or "hermes") @@ -1292,6 +1299,10 @@ def on_pre_compress(self, messages: list[dict[str, Any]]) -> str: # type: ignor model actually saw. """ with self._state_lock: + # Everything captured before this boundary is about to leave the + # raw conversation window. The shared retrieval core interprets + # this as "same-session traces older than this are eligible". + self._visible_context_start_ts = int(time.time() * 1000) context = self._last_injected_context.strip() if not context: return "" @@ -1673,6 +1684,7 @@ def on_session_end(self, messages: list[dict[str, Any]]) -> None: # type: ignor with self._state_lock: self._last_injected_context = "" self._active_turn_key = "" + self._visible_context_start_ts = None if not self._bridge: return # `sync_turn` already flushed completed turn data synchronously. @@ -2226,16 +2238,25 @@ def _turn_start(self, query: str, *, session_id: str = "") -> str: host_runtime = self._host_runtime_context() with self._state_lock: turn_key = self._active_turn_key + visible_context_start_ts = self._visible_context_start_ts + context_hints: dict[str, Any] = { + "agentIdentity": self._agent_identity, + "namespace": self._runtime_namespace(), + **host_runtime, + } + if visible_context_start_ts is not None: + context_hints.update( + { + "visibleContextKnown": True, + "visibleContextStartTs": visible_context_start_ts, + } + ) payload: dict[str, Any] = { "agent": "hermes", "namespace": self._runtime_namespace(), "sessionId": session_id or self._session_id, "userText": query, - "contextHints": { - "agentIdentity": self._agent_identity, - "namespace": self._runtime_namespace(), - **host_runtime, - }, + "contextHints": context_hints, "ts": int(time.time() * 1000), } if turn_key: diff --git a/apps/memos-local-plugin/adapters/openclaw/bridge.ts b/apps/memos-local-plugin/adapters/openclaw/bridge.ts index e3e85515f..3f5b8d676 100644 --- a/apps/memos-local-plugin/adapters/openclaw/bridge.ts +++ b/apps/memos-local-plugin/adapters/openclaw/bridge.ts @@ -250,6 +250,37 @@ export function flattenMessages(input: unknown[] | undefined): FlatMessage[] { return out; } +const MAX_VISIBLE_CONTEXT_USER_TEXTS = 128; + +/** + * Describe the conversation history OpenClaw will actually send to the + * model for this run. These hints are routing/filter metadata only; the + * retrieval query builder never includes them in semantic search text. + */ +function visibleContextHints(messages: unknown[] | undefined): Record { + const flat = flattenMessages(messages); + const timestamps = flat + .filter((message) => message.role !== "system") + .map((message) => message.ts) + .filter((ts): ts is number => typeof ts === "number" && Number.isFinite(ts)); + const userTexts = Array.from( + new Set( + flat + .filter((message) => message.role === "user") + .map((message) => message.content.trim()) + .filter(Boolean), + ), + ).slice(-MAX_VISIBLE_CONTEXT_USER_TEXTS); + + return { + visibleContextKnown: true, + ...(timestamps.length > 0 + ? { visibleContextStartTs: Math.min(...timestamps) } + : {}), + ...(userTexts.length > 0 ? { visibleContextUserTexts: userTexts } : {}), + }; +} + function isToolCallBlockType(type: string): boolean { const normalized = type.trim().toLowerCase(); return ( @@ -333,8 +364,14 @@ function extractTextContent(content: unknown): string { function pickTimestamp(m: Record): number | undefined { const candidates = [m.ts, m.timestamp, m.time, m.createdAt]; for (const c of candidates) { - if (typeof c === "number" && Number.isFinite(c)) return c; + if (typeof c === "number" && Number.isFinite(c)) { + return c > 0 && c < 10_000_000_000 ? c * 1000 : c; + } if (typeof c === "string") { + const numeric = Number(c); + if (Number.isFinite(numeric) && numeric > 0) { + return numeric < 10_000_000_000 ? numeric * 1000 : numeric; + } const parsed = Date.parse(c); if (!Number.isNaN(parsed)) return parsed; } @@ -1320,6 +1357,7 @@ export function createOpenClawBridge(opts: BridgeOptions): BridgeHandle { sessionId: ctx.sessionId, runId: ctx.runId, workspaceDir: ctx.workspaceDir, + ...visibleContextHints(event.messages), }, }; diff --git a/apps/memos-local-plugin/core/pipeline/retrieval-repos.ts b/apps/memos-local-plugin/core/pipeline/retrieval-repos.ts index 6eb60ca24..9a4773428 100644 --- a/apps/memos-local-plugin/core/pipeline/retrieval-repos.ts +++ b/apps/memos-local-plugin/core/pipeline/retrieval-repos.ts @@ -10,7 +10,7 @@ import type { RetrievalRepos } from "../retrieval/types.js"; import type { Repos } from "../storage/repos/index.js"; import type { RuntimeNamespace, TraceId } from "../../agent-contract/dto.js"; -import { isVisibleTo } from "../runtime/namespace.js"; +import { isVisibleTo, visibilityWhere } from "../runtime/namespace.js"; export function wrapRetrievalRepos(repos: Repos, namespace: RuntimeNamespace): RetrievalRepos { return { @@ -44,13 +44,25 @@ export function wrapRetrievalRepos(repos: Repos, namespace: RuntimeNamespace): R traces: { searchByVector(query, k, opts) { - return repos.traces.searchByVector(query, k, opts ?? {}); + return repos.traces.searchByVector( + query, + k, + mergeVisibility(opts, namespace), + ); }, searchByText(ftsMatch, k, opts) { - return repos.traces.searchByText(ftsMatch, k, opts ?? {}); + return repos.traces.searchByText( + ftsMatch, + k, + mergeVisibility(opts, namespace, "t"), + ); }, searchByPattern(terms, k, opts) { - return repos.traces.searchByPattern(terms, k, opts ?? {}); + return repos.traces.searchByPattern( + terms, + k, + mergeVisibility(opts, namespace), + ); }, getManyByIds(ids) { const rows = repos.traces.getManyByIds(ids as readonly TraceId[]); @@ -70,7 +82,11 @@ export function wrapRetrievalRepos(repos: Repos, namespace: RuntimeNamespace): R })); }, searchByErrorSignature(fragments, limit, opts) { - const rows = repos.traces.searchByErrorSignature(fragments, limit, opts); + const rows = repos.traces.searchByErrorSignature( + fragments, + limit, + mergeVisibility(opts, namespace), + ); return rows.filter((r) => isVisibleTo(r, namespace)).map((r) => ({ id: r.id, episodeId: r.episodeId, @@ -174,6 +190,27 @@ export function wrapRetrievalRepos(repos: Repos, namespace: RuntimeNamespace): R }; } +function mergeVisibility; +}>( + opts: T | undefined, + namespace: RuntimeNamespace, + alias = "", +): T { + const visible = visibilityWhere(namespace, alias); + return { + ...(opts ?? {}), + where: opts?.where + ? `(${opts.where}) AND (${visible.sql})` + : visible.sql, + params: { + ...(opts?.params ?? {}), + ...visible.params, + }, + } as T; +} + function normaliseSkillDecisionGuidance( procedureJson: unknown, ): { preference: string[]; antiPattern: string[] } { diff --git a/apps/memos-local-plugin/core/retrieval/retrieve.ts b/apps/memos-local-plugin/core/retrieval/retrieve.ts index 11a998349..cec076f62 100644 --- a/apps/memos-local-plugin/core/retrieval/retrieve.ts +++ b/apps/memos-local-plugin/core/retrieval/retrieve.ts @@ -37,7 +37,7 @@ import { llmFilterCandidates } from "./llm-filter.js"; import { rank } from "./ranker.js"; import { runTier1 } from "./tier1-skill.js"; import { runTier2Experience } from "./tier2-experience.js"; -import { runTier2 } from "./tier2-trace.js"; +import { runTier2, type Tier2Input } from "./tier2-trace.js"; import { runTier3 } from "./tier3-world.js"; import type { EpisodeCandidate, @@ -312,6 +312,7 @@ async function runAll( : Promise.resolve([]); const tier2Start = Date.now(); + const currentVisibleContext = resolveVisibleContext(ctx, sessionId, deps.config); const tier2Promise: Promise<{ traces: TraceCandidate[]; episodes: EpisodeCandidate[] }> = wantTier2 && !noUsableChannel ? runTier2( @@ -325,10 +326,7 @@ async function runAll( exactIdentifiers: compiled.exactIdentifiers, profile, includeLowValue: plan.includeLowValue, - excludeSessionId: - ctx.reason === "turn_start" && sessionId && !deps.config.lightweightMemory - ? sessionId - : undefined, + ...currentVisibleContext, }, ) : Promise.resolve({ traces: [], episodes: [] }); @@ -443,6 +441,7 @@ async function runAll( profile, includeActionVector: true, includeLowValue: plan.includeLowValue, + ...currentVisibleContext, }, ); tier2LatencyMs += Date.now() - wideningStartedAt; @@ -714,6 +713,49 @@ function approxTokens(s: string): number { return Math.ceil(s.length / 4); } +function resolveVisibleContext( + ctx: RetrievalCtx, + sessionId: SessionId | undefined, + config: RetrievalDeps["config"], +): Pick { + if ( + ctx.reason !== "turn_start" || + !sessionId || + config.lightweightMemory + ) { + return {}; + } + + const hints = ctx.contextHints; + if (hints?.visibleContextKnown !== true) { + // Backward compatibility for adapters that cannot yet report the + // model-visible window. This preserves the old no-repeat behaviour + // instead of guessing that the session has been compacted. + return { excludeSessionId: sessionId }; + } + + const rawStartTs = hints.visibleContextStartTs; + const startTs = + typeof rawStartTs === "number" && Number.isFinite(rawStartTs) + ? rawStartTs + : undefined; + const userTexts = Array.isArray(hints.visibleContextUserTexts) + ? hints.visibleContextUserTexts + .filter((text): text is string => typeof text === "string") + .map((text) => text.trim()) + .filter(Boolean) + .slice(-128) + : []; + + return { + visibleContext: { + sessionId, + ...(startTs !== undefined ? { startTs } : {}), + ...(userTexts.length > 0 ? { userTexts } : {}), + }, + }; +} + function round(n: number, d: number): number { if (!Number.isFinite(n)) return n; const f = 10 ** d; diff --git a/apps/memos-local-plugin/core/retrieval/tier2-trace.ts b/apps/memos-local-plugin/core/retrieval/tier2-trace.ts index a02a79f2a..437df3d82 100644 --- a/apps/memos-local-plugin/core/retrieval/tier2-trace.ts +++ b/apps/memos-local-plugin/core/retrieval/tier2-trace.ts @@ -69,8 +69,22 @@ export interface Tier2Input { /** Whether `decision_repair` forced `includeLowValue`. */ includeLowValue?: boolean; /** - * When set, trace search excludes rows from this session (cross-session - * turn-start retrieval should not repeat the current chat window). + * The portion of the current durable session that the host model can + * still see. Rows inside this window are filtered before channel Top-K; + * rows older than `startTs` remain eligible after host compaction. + * + * `userTexts` is a bounded exact-text fallback for hosts whose visible + * messages do not carry timestamps. It is intentionally not semantic: + * embedding similarity belongs to MMR, not context-window dedupe. + */ + visibleContext?: { + sessionId: SessionId; + startTs?: number; + userTexts?: readonly string[]; + }; + /** + * Legacy fallback for hosts that cannot report their visible context. + * New adapters should send `visibleContext` instead. */ excludeSessionId?: SessionId; /** Query-specific policy; personal facts use summary vectors first. */ @@ -332,7 +346,36 @@ function buildTraceSearchFilters( const params: Record = {}; const includeLow = input.includeLowValue ?? deps.config.includeLowValue; if (!includeLow) parts.push("priority > 0"); - if (input.excludeSessionId) { + const visible = input.visibleContext; + if ( + visible && + typeof visible.startTs === "number" && + Number.isFinite(visible.startTs) + ) { + parts.push( + "(session_id != @visible_context_session_id OR ts < @visible_context_start_ts)", + ); + params.visible_context_session_id = visible.sessionId; + params.visible_context_start_ts = visible.startTs; + } else if (visible) { + const texts = Array.from( + new Set( + (visible.userTexts ?? []) + .map((text) => String(text).trim()) + .filter(Boolean), + ), + ).slice(-128); + if (texts.length > 0) { + const placeholders = texts.map((_, index) => `@visible_user_text_${index}`); + texts.forEach((text, index) => { + params[`visible_user_text_${index}`] = text; + }); + parts.push( + `(session_id != @visible_context_session_id OR user_text NOT IN (${placeholders.join(", ")}))`, + ); + params.visible_context_session_id = visible.sessionId; + } + } else if (input.excludeSessionId) { parts.push("session_id != @exclude_session_id"); params.exclude_session_id = input.excludeSessionId; } diff --git a/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py b/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py index 7ecaccb89..30b72d641 100644 --- a/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py +++ b/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py @@ -442,6 +442,31 @@ def test_on_pre_compress_without_cached_prefetch_is_read_only(self) -> None: self.assertEqual(snapshot, "") self.assertFalse(any(method == "turn.start" for method, _ in bridge.calls)) + def test_compression_boundary_is_forwarded_to_next_turn_retrieval(self) -> None: + bridge = FakeBridge() + with ( + patch("memos_provider.ensure_bridge_running", return_value=True), + patch("memos_provider.ensure_viewer_daemon", return_value=True), + patch("memos_provider.MemosBridgeClient", return_value=bridge), + patch("memos_provider.time.time", return_value=1_700_000_123.456), + ): + provider = memos_provider.MemTensorProvider() + provider.initialize("compress-session") + provider.on_pre_compress([{"role": "user", "content": "old visible turn"}]) + provider.on_turn_start(3, "what did we discuss before compression?") + provider.prefetch("what did we discuss before compression?") + + turn_start = next( + payload + for method, payload in reversed(bridge.calls) + if method == "turn.start" + ) + self.assertEqual( + turn_start["contextHints"]["visibleContextStartTs"], + 1_700_000_123_456, + ) + self.assertTrue(turn_start["contextHints"]["visibleContextKnown"]) + def test_prefetch_passes_stable_turn_key_to_bridge(self) -> None: bridge = FakeBridge() with ( diff --git a/apps/memos-local-plugin/tests/unit/adapters/openclaw-bridge.test.ts b/apps/memos-local-plugin/tests/unit/adapters/openclaw-bridge.test.ts index 1adb51081..d6b638597 100644 --- a/apps/memos-local-plugin/tests/unit/adapters/openclaw-bridge.test.ts +++ b/apps/memos-local-plugin/tests/unit/adapters/openclaw-bridge.test.ts @@ -1152,6 +1152,42 @@ describe("createOpenClawBridge", () => { } }); + it("passes the model-visible conversation window as retrieval-only context hints", async () => { + const mc = buildCore(); + await mc.init(); + const onTurnStart = vi.spyOn(mc, "onTurnStart"); + + const bridge = createOpenClawBridge({ + agent: "openclaw", + core: mc, + log: silentLogger(), + }); + await bridge.handleBeforePrompt( + { + prompt: "你还记得我喜欢什么吗", + messages: [ + { + role: "user", + content: "[message_id: old]\nou_sender: 我喜欢苹果", + timestamp: 1_700_000_100_000, + }, + { + role: "assistant", + content: "记住了", + timestamp: 1_700_000_101_000, + }, + ], + }, + hookCtx({ sessionKey: "feishu:single-session" }), + ); + + expect(onTurnStart).toHaveBeenCalledOnce(); + expect(onTurnStart.mock.calls[0]![0].contextHints).toMatchObject({ + visibleContextStartTs: 1_700_000_100_000, + visibleContextUserTexts: ["我喜欢苹果"], + }); + }); + it("handleAgentEnd feeds onTurnEnd and produces trace + episode.closed events", async () => { const mc = buildCore(); await mc.init(); diff --git a/apps/memos-local-plugin/tests/unit/pipeline/retrieval-repos.test.ts b/apps/memos-local-plugin/tests/unit/pipeline/retrieval-repos.test.ts new file mode 100644 index 000000000..a048899d7 --- /dev/null +++ b/apps/memos-local-plugin/tests/unit/pipeline/retrieval-repos.test.ts @@ -0,0 +1,92 @@ +import { afterEach, describe, expect, it } from "vitest"; + +import { wrapRetrievalRepos } from "../../../core/pipeline/retrieval-repos.js"; +import type { + EmbeddingVector, + EpisodeId, + SessionId, + TraceId, + TraceRow, +} from "../../../core/types.js"; +import { makeTmpDb, type TmpDbHandle } from "../../helpers/tmp-db.js"; + +function vector(values: number[]): EmbeddingVector { + return Float32Array.from(values); +} + +describe("pipeline/retrieval-repos", () => { + let handle: TmpDbHandle | null = null; + + afterEach(() => { + handle?.cleanup(); + handle = null; + }); + + it("applies trace namespace visibility before channel Top-K", () => { + handle = makeTmpDb({ agent: "openclaw" }); + const now = 1_700_000_000_000; + + for (const profileId of ["main", "other"]) { + const sessionId = `session-${profileId}` as SessionId; + const episodeId = `episode-${profileId}` as EpisodeId; + handle.repos.sessions.upsert({ + id: sessionId, + agent: "openclaw", + ownerAgentKind: "openclaw", + ownerProfileId: profileId, + ownerWorkspaceId: null, + startedAt: now, + lastSeenAt: now, + meta: {}, + }); + handle.repos.episodes.upsert({ + id: episodeId, + sessionId, + ownerAgentKind: "openclaw", + ownerProfileId: profileId, + ownerWorkspaceId: null, + startedAt: now, + endedAt: null, + traceIds: [], + rTask: null, + status: "open", + }); + handle.repos.traces.insert({ + id: `trace-${profileId}` as TraceId, + episodeId, + sessionId, + ownerAgentKind: "openclaw", + ownerProfileId: profileId, + ownerWorkspaceId: null, + ts: now, + userText: `${profileId} fact`, + agentText: "reply", + summary: null, + share: null, + toolCalls: [], + agentThinking: null, + reflection: null, + value: 0.8, + alpha: 0.5, + rHuman: null, + priority: 0.8, + tags: [], + errorSignatures: [], + vecSummary: profileId === "other" ? vector([1, 0]) : vector([0.9, 0.1]), + vecAction: null, + turnId: now, + schemaVersion: 1, + } as TraceRow); + } + + const unwrapped = handle.repos.traces.searchByVector(vector([1, 0]), 1); + expect(unwrapped[0]?.id).toBe("trace-other"); + + const wrapped = wrapRetrievalRepos(handle.repos, { + agentKind: "openclaw", + profileId: "main", + }); + const visible = wrapped.traces.searchByVector(vector([1, 0]), 1); + expect(visible[0]?.id).toBe("trace-main"); + }); +}); diff --git a/apps/memos-local-plugin/tests/unit/retrieval/integration.test.ts b/apps/memos-local-plugin/tests/unit/retrieval/integration.test.ts index 7bd8c85fa..f49b7e198 100644 --- a/apps/memos-local-plugin/tests/unit/retrieval/integration.test.ts +++ b/apps/memos-local-plugin/tests/unit/retrieval/integration.test.ts @@ -231,6 +231,58 @@ describe("retrieval/integration", () => { expect(skillIds).not.toContain("sk_weak"); }); + it("turn_start recalls same-session traces only from before the visible window", async () => { + handle.repos.traces.insert({ + id: "t_before_compaction" as TraceId, + episodeId: "ep1" as EpisodeId, + sessionId: "s1" as SessionId, + ts: (NOW - 10_000) as never, + userText: "run docker compose before compaction", + agentText: "agent text before compaction", + toolCalls: [], + reflection: "old relevant trace", + value: 0.9 as never, + alpha: 0.5 as never, + rHuman: null, + priority: 0.9 as never, + tags: ["docker"], + vecSummary: vec([1, 0, 0]), + vecAction: null, + turnId: 1 as never, + schemaVersion: 1, + }); + + const res = await turnStartRetrieve( + makeDeps(handle), + { + reason: "turn_start", + agent: "openclaw", + sessionId: "s1" as SessionId, + userText: "run docker compose", + contextHints: { + visibleContextKnown: true, + visibleContextStartTs: NOW - 5_000, + visibleContextUserTexts: ["user text t_hi"], + }, + ts: NOW as never, + }, + { + plan: { + wantTier1: false, + wantTier2: true, + wantTier3: false, + }, + }, + ); + + const traceIds = res.packet.snippets + .filter((snippet) => snippet.refKind === "trace") + .map((snippet) => String(snippet.refId)); + expect(traceIds).toContain("t_before_compaction"); + expect(traceIds).not.toContain("t_hi"); + expect(traceIds).not.toContain("t_med"); + }); + it("keeps abstract memories when long unique identifier queries require keywords", async () => { const res = await turnStartRetrieve(makeDeps(handle), { reason: "turn_start", diff --git a/apps/memos-local-plugin/tests/unit/retrieval/tier2.test.ts b/apps/memos-local-plugin/tests/unit/retrieval/tier2.test.ts index 5d7fae5cb..50b8433ef 100644 --- a/apps/memos-local-plugin/tests/unit/retrieval/tier2.test.ts +++ b/apps/memos-local-plugin/tests/unit/retrieval/tier2.test.ts @@ -303,7 +303,7 @@ describe("retrieval/tier2 (with real sqlite)", () => { expect(ids).toContain("zeroV"); }); - it("excludeSessionId omits traces from that session", async () => { + it("excludes only the visible window of the current session", async () => { handle.repos.sessions.upsert({ id: "s2" as SessionId, agent: "openclaw", @@ -321,11 +321,11 @@ describe("retrieval/tier2 (with real sqlite)", () => { status: "open", }); handle.repos.traces.insert({ - id: "otherSess" as TraceId, + id: "compactedSameSession" as TraceId, episodeId: "ep2" as EpisodeId, sessionId: "s2" as SessionId, - ts: NOW as never, - userText: "other session docker query", + ts: (NOW - 10_000) as never, + userText: "compacted same session docker query", agentText: "reply", toolCalls: [], reflection: "ref", @@ -339,22 +339,64 @@ describe("retrieval/tier2 (with real sqlite)", () => { turnId: 0 as never, schemaVersion: 1, }); + handle.repos.traces.insert({ + id: "visibleSameSession" as TraceId, + episodeId: "ep2" as EpisodeId, + sessionId: "s2" as SessionId, + ts: NOW as never, + userText: "visible same session docker query", + agentText: "reply", + toolCalls: [], + reflection: "ref", + value: 0.95 as never, + alpha: 0.5 as never, + rHuman: null, + priority: 0.95 as never, + tags: ["docker"], + vecSummary: vec([1, 0, 0]), + vecAction: null, + turnId: 1 as never, + schemaVersion: 1, + }); - const withoutExclude = await runTier2( + const withoutWindow = await runTier2( { repos: { traces: handle.repos.traces }, config: cfg, now: () => NOW }, { queryVec: vec([1, 0, 0]), tags: ["docker"] }, ); - expect(withoutExclude.traces.map((t) => String(t.refId))).toContain("otherSess"); + expect(withoutWindow.traces.map((t) => String(t.refId))).toEqual( + expect.arrayContaining(["compactedSameSession", "visibleSameSession"]), + ); - const withExclude = await runTier2( + const withWindow = await runTier2( + { repos: { traces: handle.repos.traces }, config: cfg, now: () => NOW }, + { + queryVec: vec([1, 0, 0]), + tags: ["docker"], + visibleContext: { + sessionId: "s2" as SessionId, + startTs: NOW - 5_000, + userTexts: ["visible same session docker query"], + }, + }, + ); + const ids = withWindow.traces.map((t) => String(t.refId)); + expect(ids).toContain("compactedSameSession"); + expect(ids).not.toContain("visibleSameSession"); + }); + + it("falls back to exact visible-user-text filtering when timestamps are unavailable", async () => { + const out = await runTier2( { repos: { traces: handle.repos.traces }, config: cfg, now: () => NOW }, { queryVec: vec([1, 0, 0]), tags: ["docker"], - excludeSessionId: "s2" as SessionId, + visibleContext: { + sessionId: "s1" as SessionId, + userTexts: ["hiV query about docker"], + }, }, ); - expect(withExclude.traces.map((t) => String(t.refId))).not.toContain("otherSess"); + expect(out.traces.map((t) => String(t.refId))).not.toContain("hiV"); }); it("rolls up episodes when ≥2 traces share episode_id", async () => { From 10c15a04a177dcb1e87ad0791297c8ccddab4926 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=81=E5=9C=A8=E5=90=B5=E7=9D=80=E5=90=83=E7=B3=96?= Date: Mon, 3 Aug 2026 15:19:06 +0800 Subject: [PATCH 13/18] style(plugin): format Hermes provider pipeline test --- .../tests/python/test_hermes_provider_pipeline.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py b/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py index 30b72d641..6b95ae61b 100644 --- a/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py +++ b/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py @@ -457,9 +457,7 @@ def test_compression_boundary_is_forwarded_to_next_turn_retrieval(self) -> None: provider.prefetch("what did we discuss before compression?") turn_start = next( - payload - for method, payload in reversed(bridge.calls) - if method == "turn.start" + payload for method, payload in reversed(bridge.calls) if method == "turn.start" ) self.assertEqual( turn_start["contextHints"]["visibleContextStartTs"], From c14c3e680f00e3240b472176c897b582d8e27858 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=81=E5=9C=A8=E5=90=B5=E7=9D=80=E5=90=83=E7=B3=96?= Date: Tue, 4 Aug 2026 16:25:28 +0800 Subject: [PATCH 14/18] fix(plugin): bound foreground retries and shutdown --- .../hermes/memos_provider/__init__.py | 148 +++++++++- apps/memos-local-plugin/agent-contract/dto.ts | 5 + apps/memos-local-plugin/bridge/methods.ts | 9 + .../core/embedding/embedder.ts | 10 +- .../core/embedding/fetcher.ts | 23 +- .../core/embedding/index.ts | 1 + .../core/embedding/retry-worker.ts | 8 +- .../core/embedding/types.ts | 11 +- apps/memos-local-plugin/core/index.ts | 1 + apps/memos-local-plugin/core/llm/fetcher.ts | 32 +- apps/memos-local-plugin/core/pipeline/deps.ts | 29 +- .../core/pipeline/orchestrator.ts | 153 ++++++++-- .../core/retrieval/llm-filter.ts | 6 + .../core/retrieval/retrieve.ts | 48 ++- .../core/retrieval/types.ts | 6 +- .../core/session/intent-classifier.ts | 8 +- .../core/session/manager.ts | 3 + .../core/session/relation-classifier.ts | 24 +- apps/memos-local-plugin/core/session/types.ts | 2 + .../core/util/foreground-resources.ts | 274 ++++++++++++++++++ .../core/util/rate-limited-llm.ts | 31 +- .../core/util/request-deadline.ts | 37 +++ .../core/util/retry-after.ts | 61 ++++ .../memos-local-plugin/core/util/semaphore.ts | 43 ++- .../tests/python/test_bridge_client.py | 43 ++- .../python/test_hermes_provider_pipeline.py | 69 +++++ .../tests/unit/bridge/methods.test.ts | 11 + .../tests/unit/embedding/embedder.test.ts | 17 ++ .../tests/unit/embedding/fetcher.test.ts | 27 ++ .../tests/unit/embedding/retry-worker.test.ts | 26 ++ .../tests/unit/llm/fetcher.test.ts | 73 ++++- .../tests/unit/pipeline/orchestrator.test.ts | 33 +++ .../unit/session/intent-classifier.test.ts | 19 ++ .../unit/session/relation-classifier.test.ts | 27 ++ .../unit/util/foreground-resources.test.ts | 138 +++++++++ .../tests/unit/util/request-deadline.test.ts | 35 +++ .../tests/unit/util/retry-after.test.ts | 34 +++ .../tests/unit/util/semaphore.test.ts | 19 ++ 38 files changed, 1435 insertions(+), 109 deletions(-) create mode 100644 apps/memos-local-plugin/core/util/foreground-resources.ts create mode 100644 apps/memos-local-plugin/core/util/request-deadline.ts create mode 100644 apps/memos-local-plugin/core/util/retry-after.ts create mode 100644 apps/memos-local-plugin/tests/unit/util/foreground-resources.test.ts create mode 100644 apps/memos-local-plugin/tests/unit/util/request-deadline.test.ts create mode 100644 apps/memos-local-plugin/tests/unit/util/retry-after.test.ts create mode 100644 apps/memos-local-plugin/tests/unit/util/semaphore.test.ts diff --git a/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py b/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py index 6eeaa000e..6353c560a 100644 --- a/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py +++ b/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py @@ -201,6 +201,46 @@ def _long_rpc_timeout_default() -> float: _LONG_RPC_TIMEOUT = _long_rpc_timeout_default() + +def _prefetch_rpc_timeout_default() -> float: + """Resolve the latency budget for Hermes' foreground memory lookup. + + Hermes places its own short deadline around ``prefetch``. Reusing the + long capture timeout here lets the bridge continue work after the host + has already moved on. Keep a separate, configurable ceiling below the + host's default and forward the corresponding absolute deadline to core. + """ + raw = os.environ.get("MEMOS_HERMES_PREFETCH_RPC_TIMEOUT", "") + try: + value = float(raw) + except (TypeError, ValueError): + return 6.0 + if not value > 0: + return 6.0 + # Hermes currently abandons external providers after 8 seconds. Keep at + # least one second for Python thread scheduling and response assembly even + # when an operator overrides the default. + return min(value, 7.0) + + +_PREFETCH_RPC_TIMEOUT = _prefetch_rpc_timeout_default() +_PREFETCH_RESPONSE_RESERVE_SECONDS = 0.25 + + +def _remaining_rpc_timeout( + deadline_monotonic: float | None, + requested_timeout: float | None, +) -> float | None: + """Bound one blocking bridge step by a shared end-to-end deadline.""" + if deadline_monotonic is None: + return requested_timeout + remaining = deadline_monotonic - time.monotonic() + if remaining <= 0: + raise BridgeError("timeout", "foreground prefetch deadline exceeded") + if requested_timeout is None: + return remaining + return min(requested_timeout, remaining) + _HERMES_INTERNAL_REVIEW_PREFIXES = ( "review the conversation above and consider saving to memory if appropriate.", "review the conversation above and update the skill library.", @@ -1022,8 +1062,19 @@ def prefetch(self, query: str, *, session_id: str = "") -> str: # type: ignore[ cached result immediately. Otherwise synchronously run ``turn.start`` against the bridge (small overhead). """ + deadline_monotonic = time.monotonic() + _PREFETCH_RPC_TIMEOUT + started_at_ms = int(time.time() * 1000) + core_budget_seconds = max( + 0.05, + _PREFETCH_RPC_TIMEOUT - _PREFETCH_RESPONSE_RESERVE_SECONDS, + ) + deadline_at_ms = started_at_ms + int(core_budget_seconds * 1000) if self._prefetch_thread and self._prefetch_thread.is_alive(): - self._prefetch_thread.join(timeout=5.0) + try: + join_timeout = _remaining_rpc_timeout(deadline_monotonic, 5.0) + except BridgeError: + return "" + self._prefetch_thread.join(timeout=join_timeout) with self._prefetch_lock: cached = self._prefetch_result self._prefetch_result = "" @@ -1033,10 +1084,25 @@ def prefetch(self, query: str, *, session_id: str = "") -> str: # type: ignore[ suppress_injection = _is_explicit_delegation_request(query) if cached: return "" if suppress_injection else cached - if not self._ensure_bridge(session_id or self._session_id, timeout=10.0): + try: + ensure_timeout = _remaining_rpc_timeout( + deadline_monotonic, + _PREFETCH_RPC_TIMEOUT, + ) + except BridgeError: + return "" + if not self._ensure_bridge( + session_id or self._session_id, + timeout=min(10.0, ensure_timeout or _PREFETCH_RPC_TIMEOUT), + ): return "" try: - context = self._turn_start(query, session_id=session_id) + context = self._turn_start( + query, + session_id=session_id, + deadline_monotonic=deadline_monotonic, + deadline_at_ms=deadline_at_ms, + ) if suppress_injection: # Do not let remembered "do it directly" skills override an # explicit user request to dispatch work to a subagent. @@ -1941,6 +2007,7 @@ def _bridge_request( *, timeout: float | None = None, ensure_session: bool = True, + deadline_monotonic: float | None = None, ) -> dict[str, Any]: bridge = self._bridge if bridge is None: @@ -1958,10 +2025,23 @@ def _bridge_request( bridge.generation, self._session_id, ) - self._open_session(self._session_id, timeout=30.0) - if timeout is None: + session_ceiling = ( + timeout + if deadline_monotonic is not None and timeout is not None + else 30.0 + ) + session_timeout = _remaining_rpc_timeout( + deadline_monotonic, + session_ceiling, + ) + self._open_session( + self._session_id, + timeout=session_timeout or 30.0, + ) + request_timeout = _remaining_rpc_timeout(deadline_monotonic, timeout) + if request_timeout is None: return bridge.request(method, params) - return bridge.request(method, params, timeout=timeout) + return bridge.request(method, params, timeout=request_timeout) def _open_session(self, session_id: str = "", *, timeout: float = 30.0) -> None: bridge = self._bridge @@ -1997,6 +2077,7 @@ def _bridge_request_with_retry( params: Any, *, timeout: float | None = None, + deadline_monotonic: float | None = None, ) -> dict[str, Any]: """Read-path helper: reconnect + retry once on ``transport_closed``. @@ -2012,7 +2093,12 @@ def _bridge_request_with_retry( """ assert self._bridge is not None try: - return self._bridge_request(method, params, timeout=timeout) + return self._bridge_request( + method, + params, + timeout=timeout, + deadline_monotonic=deadline_monotonic, + ) except BridgeError as err: if not self._is_transport_closed(err): raise @@ -2021,9 +2107,26 @@ def _bridge_request_with_retry( method, err, ) - self._reconnect_bridge(self._session_id, timeout=30.0) + reconnect_ceiling = ( + timeout + if deadline_monotonic is not None and timeout is not None + else 30.0 + ) + reconnect_timeout = _remaining_rpc_timeout( + deadline_monotonic, + reconnect_ceiling, + ) + self._reconnect_bridge( + self._session_id, + timeout=reconnect_timeout or 30.0, + ) assert self._bridge is not None - return self._bridge_request(method, params, timeout=timeout) + return self._bridge_request( + method, + params, + timeout=timeout, + deadline_monotonic=deadline_monotonic, + ) def _is_transport_closed(self, err: Exception) -> bool: if isinstance(err, BridgeError) and err.code == "transport_closed": @@ -2233,7 +2336,14 @@ def _run() -> None: ) self._bridge_keepalive_thread.start() - def _turn_start(self, query: str, *, session_id: str = "") -> str: + def _turn_start( + self, + query: str, + *, + session_id: str = "", + deadline_monotonic: float | None = None, + deadline_at_ms: int | None = None, + ) -> str: assert self._bridge is not None host_runtime = self._host_runtime_context() with self._state_lock: @@ -2251,20 +2361,34 @@ def _turn_start(self, query: str, *, session_id: str = "") -> str: "visibleContextStartTs": visible_context_start_ts, } ) + now_ms = int(time.time() * 1000) + if deadline_monotonic is None: + deadline_monotonic = time.monotonic() + _PREFETCH_RPC_TIMEOUT + if deadline_at_ms is None: + core_budget_seconds = max( + 0.05, + _PREFETCH_RPC_TIMEOUT - _PREFETCH_RESPONSE_RESERVE_SECONDS, + ) + deadline_at_ms = now_ms + int(core_budget_seconds * 1000) payload: dict[str, Any] = { "agent": "hermes", "namespace": self._runtime_namespace(), "sessionId": session_id or self._session_id, "userText": query, "contextHints": context_hints, - "ts": int(time.time() * 1000), + "ts": now_ms, + "deadlineAt": deadline_at_ms, } if turn_key: payload["turnKey"] = turn_key resp = self._bridge_request_with_retry( "turn.start", payload, - timeout=_LONG_RPC_TIMEOUT, + timeout=_remaining_rpc_timeout( + deadline_monotonic, + _PREFETCH_RPC_TIMEOUT, + ), + deadline_monotonic=deadline_monotonic, ) response_query = (resp or {}).get("query") or {} response_session = str(response_query.get("sessionId") or "") diff --git a/apps/memos-local-plugin/agent-contract/dto.ts b/apps/memos-local-plugin/agent-contract/dto.ts index e9ae71101..b76232bee 100644 --- a/apps/memos-local-plugin/agent-contract/dto.ts +++ b/apps/memos-local-plugin/agent-contract/dto.ts @@ -103,6 +103,11 @@ export interface TurnInputDTO { contextHints?: Record; /** Wall-clock when the turn began. */ ts: EpochMs; + /** + * Absolute adapter deadline for foreground work. Every pipeline stage + * shares this budget; it is not reset after relation or intent handling. + */ + deadlineAt?: EpochMs; } export interface TurnResultDTO { diff --git a/apps/memos-local-plugin/bridge/methods.ts b/apps/memos-local-plugin/bridge/methods.ts index 2bc1084a6..1e6cc27a0 100644 --- a/apps/memos-local-plugin/bridge/methods.ts +++ b/apps/memos-local-plugin/bridge/methods.ts @@ -389,6 +389,15 @@ function validateTurnInput(p: Record): void { "turn.start: optional 'turnKey' must be a string", ); } + if ( + p.deadlineAt !== undefined && + (typeof p.deadlineAt !== "number" || !Number.isFinite(p.deadlineAt)) + ) { + throw new MemosError( + "invalid_argument", + "turn.start: optional 'deadlineAt' must be a finite number", + ); + } } function validateTurnResult(p: Record): void { diff --git a/apps/memos-local-plugin/core/embedding/embedder.ts b/apps/memos-local-plugin/core/embedding/embedder.ts index bd4a5fe92..0b21209f4 100644 --- a/apps/memos-local-plugin/core/embedding/embedder.ts +++ b/apps/memos-local-plugin/core/embedding/embedder.ts @@ -32,6 +32,7 @@ import { MistralEmbeddingProvider } from "./providers/mistral.js"; import { OpenAiEmbeddingProvider } from "./providers/openai.js"; import { VoyageEmbeddingProvider } from "./providers/voyage.js"; import type { + EmbedCallOptions, EmbedInput, EmbedRole, EmbedStats, @@ -96,13 +97,17 @@ export function createEmbedderWithProvider( } } - async function embedOne(input: string | EmbedInput): Promise { - const vecs = await embedMany([input]); + async function embedOne( + input: string | EmbedInput, + options?: EmbedCallOptions, + ): Promise { + const vecs = await embedMany([input], options); return vecs[0]!; } async function embedMany( inputs: Array, + options?: EmbedCallOptions, ): Promise { requests += inputs.length; if (inputs.length === 0) return []; @@ -179,6 +184,7 @@ export function createEmbedderWithProvider( const ctx: ProviderCallCtx = { config, log: providerCtxLog, + signal: options?.signal, }; raw = await provider.embed(texts, role, ctx); // Record success but DO NOT clear `lastError` — the viewer diff --git a/apps/memos-local-plugin/core/embedding/fetcher.ts b/apps/memos-local-plugin/core/embedding/fetcher.ts index 40524aac7..c17b676a6 100644 --- a/apps/memos-local-plugin/core/embedding/fetcher.ts +++ b/apps/memos-local-plugin/core/embedding/fetcher.ts @@ -8,6 +8,7 @@ */ import { ERROR_CODES, MemosError } from "../../agent-contract/errors.js"; +import { parseRetryAfterMs, retryDelayMs, waitForRetry } from "../util/retry-after.js"; import type { EmbeddingProviderName, ProviderLogger } from "./types.js"; export interface HttpPostOpts { @@ -46,15 +47,19 @@ export async function httpPostJson(opts: HttpPostOpts): Promise< if (!resp.ok) { const text = await safeText(resp); const transient = resp.status >= 500 || resp.status === 429; + const retryAfterMs = resp.status === 429 || resp.status === 503 + ? parseRetryAfterMs(resp.headers.get("Retry-After")) + : null; opts.log.warn("http.non_ok", { url: opts.url, status: resp.status, attempt, transient, + retryAfterMs, durationMs: Date.now() - start, }); if (transient && attempt <= maxRetries) { - await backoff(attempt); + await backoff(attempt, retryAfterMs, opts.signal); continue; } throw new MemosError( @@ -83,7 +88,7 @@ export async function httpPostJson(opts: HttpPostOpts): Promise< durationMs: Date.now() - start, }); if (transient && attempt <= maxRetries) { - await backoff(attempt); + await backoff(attempt, null, opts.signal); continue; } throw new MemosError( @@ -123,11 +128,15 @@ function isTransientError(err: unknown): boolean { return false; } -async function backoff(attempt: number): Promise { - const base = 200; - const jitter = Math.floor(Math.random() * 100); - const ms = base * 2 ** (attempt - 1) + jitter; - await new Promise((r) => setTimeout(r, ms)); +async function backoff( + attempt: number, + retryAfterMs: number | null, + signal?: AbortSignal, +): Promise { + await waitForRetry( + retryDelayMs({ attempt, baseMs: 200, jitterMaxMs: 100, retryAfterMs }), + signal, + ); } function mergeSignals(a: AbortSignal | undefined, b: AbortSignal): AbortSignal { diff --git a/apps/memos-local-plugin/core/embedding/index.ts b/apps/memos-local-plugin/core/embedding/index.ts index 99faa0048..f6f4b1ed9 100644 --- a/apps/memos-local-plugin/core/embedding/index.ts +++ b/apps/memos-local-plugin/core/embedding/index.ts @@ -19,6 +19,7 @@ export { l2Normalize, enforceDim, postProcess, toFloat32 } from "./normalize.js" export { createEmbeddingRetryWorker, systemErrorEvent } from "./retry-worker.js"; export type { EmbeddingRetryWorker } from "./retry-worker.js"; export type { + EmbedCallOptions, EmbedInput, EmbedRole, EmbedStats, diff --git a/apps/memos-local-plugin/core/embedding/retry-worker.ts b/apps/memos-local-plugin/core/embedding/retry-worker.ts index 3620fd9e3..3c33c2870 100644 --- a/apps/memos-local-plugin/core/embedding/retry-worker.ts +++ b/apps/memos-local-plugin/core/embedding/retry-worker.ts @@ -35,6 +35,7 @@ export function createEmbeddingRetryWorker( const workerId = `embedding-retry-${ids.span()}`; let timer: ReturnType | null = null; let running: Promise | null = null; + let stopped = false; async function runOnce(): Promise { if (!deps.embedder) return; @@ -182,7 +183,7 @@ export function createEmbeddingRetryWorker( } function tick(): void { - if (running) return; + if (stopped || running) return; running = runOnce().finally(() => { running = null; }); @@ -190,16 +191,17 @@ export function createEmbeddingRetryWorker( return { start(): void { - if (timer || !deps.embedder) return; + if (stopped || timer || !deps.embedder) return; tick(); timer = setInterval(tick, deps.intervalMs ?? DEFAULT_INTERVAL_MS); }, stop(): void { + stopped = true; if (timer) clearInterval(timer); timer = null; }, async flush(): Promise { - tick(); + if (!stopped) tick(); if (running) await running; }, }; diff --git a/apps/memos-local-plugin/core/embedding/types.ts b/apps/memos-local-plugin/core/embedding/types.ts index 4f3f5eb99..0c548cc92 100644 --- a/apps/memos-local-plugin/core/embedding/types.ts +++ b/apps/memos-local-plugin/core/embedding/types.ts @@ -166,13 +166,16 @@ export interface Embedder { /** Model identifier as configured by the operator (e.g. "bge-m3"). */ readonly model: string; - embedOne(input: string | EmbedInput): Promise; + embedOne(input: string | EmbedInput, options?: EmbedCallOptions): Promise; /** * Batch-embed many texts. Results keep input order. Duplicates are deduped * internally so a text repeated N times causes 1 cache miss max. */ - embedMany(inputs: Array): Promise; + embedMany( + inputs: Array, + options?: EmbedCallOptions, + ): Promise; stats(): EmbedStats; @@ -181,6 +184,10 @@ export interface Embedder { close(): Promise; } +export interface EmbedCallOptions { + signal?: AbortSignal; +} + // ─── Errors ────────────────────────────────────────────────────────────────── export interface ProviderHttpFailure { diff --git a/apps/memos-local-plugin/core/index.ts b/apps/memos-local-plugin/core/index.ts index d2b979bce..c5e4fab9d 100644 --- a/apps/memos-local-plugin/core/index.ts +++ b/apps/memos-local-plugin/core/index.ts @@ -110,6 +110,7 @@ export { MistralEmbeddingProvider, type EmbedCache, type EmbedCacheStats, + type EmbedCallOptions, type EmbedInput, type EmbedRole, type EmbedStats, diff --git a/apps/memos-local-plugin/core/llm/fetcher.ts b/apps/memos-local-plugin/core/llm/fetcher.ts index 53eb55ec3..ef0d56357 100644 --- a/apps/memos-local-plugin/core/llm/fetcher.ts +++ b/apps/memos-local-plugin/core/llm/fetcher.ts @@ -11,6 +11,7 @@ */ import { ERROR_CODES, MemosError } from "../../agent-contract/errors.js"; +import { parseRetryAfterMs, retryDelayMs, waitForRetry } from "../util/retry-after.js"; import type { LlmProviderLogger, LlmProviderName } from "./types.js"; export interface HttpPostOpts { @@ -56,16 +57,20 @@ export async function httpPostJson(opts: HttpPostOpts): Promise< if (!resp.ok) { const text = await safeText(resp); const transient = resp.status >= 500 || resp.status === 429; + const retryAfterMs = resp.status === 429 || resp.status === 503 + ? parseRetryAfterMs(resp.headers.get("Retry-After")) + : null; opts.log.warn("http.non_ok", { status: resp.status, attempt, transient, durationMs: ms, + retryAfterMs, body: truncateLogBody(text), }); if (transient && attempt <= opts.maxRetries) { opts.onRetry?.(attempt); - await backoff(attempt); + await backoff(attempt, retryAfterMs, opts.signal); continue; } throw new MemosError( @@ -85,8 +90,15 @@ export async function httpPostJson(opts: HttpPostOpts): Promise< } catch (err) { lastErr = err; if (err instanceof MemosError) throw err; + if (opts.signal?.aborted) { + throw new MemosError( + ERROR_CODES.LLM_TIMEOUT, + `${opts.provider} request was cancelled`, + { provider: opts.provider, url: opts.url, cancelled: true }, + ); + } const transient = isTransientError(err); - const timedOut = isTimeout(err); + const timedOut = isTimeout(err) || opts.signal?.aborted === true; opts.log.warn("http.exception", { attempt, transient, @@ -95,7 +107,7 @@ export async function httpPostJson(opts: HttpPostOpts): Promise< }); if ((transient || timedOut) && attempt <= opts.maxRetries) { opts.onRetry?.(attempt); - await backoff(attempt); + await backoff(attempt, null, opts.signal); continue; } if (timedOut) { @@ -245,11 +257,15 @@ function isTimeout(err: unknown): boolean { return false; } -async function backoff(attempt: number): Promise { - const base = 250; - const jitter = Math.floor(Math.random() * 120); - const ms = base * 2 ** (attempt - 1) + jitter; - await new Promise((r) => setTimeout(r, ms)); +async function backoff( + attempt: number, + retryAfterMs: number | null, + signal?: AbortSignal, +): Promise { + await waitForRetry( + retryDelayMs({ attempt, baseMs: 250, jitterMaxMs: 120, retryAfterMs }), + signal, + ); } function mergeSignals(a: AbortSignal | undefined, b: AbortSignal): AbortSignal { diff --git a/apps/memos-local-plugin/core/pipeline/deps.ts b/apps/memos-local-plugin/core/pipeline/deps.ts index 8a0772119..79714b35e 100644 --- a/apps/memos-local-plugin/core/pipeline/deps.ts +++ b/apps/memos-local-plugin/core/pipeline/deps.ts @@ -101,6 +101,10 @@ import type { import { wrapRetrievalRepos } from "./retrieval-repos.js"; import { createSemaphore } from "../util/semaphore.js"; import { rateLimitLlmClient } from "../util/rate-limited-llm.js"; +import { + prioritizeEmbedder, + type ForegroundResources, +} from "../util/foreground-resources.js"; // ─── Algorithm config slice helper ──────────────────────────────────────── @@ -208,19 +212,23 @@ export function buildPipelineSubscribers( buses: PipelineBuses, algorithm: PipelineAlgorithmConfig, session?: PipelineSessionSet, + resources?: ForegroundResources, ): PipelineSubscriberSet { const log = deps.log ?? rootLogger.child({ channel: "core.pipeline" }); const bgLlmSemaphore = createSemaphore(algorithm.session.bgLlmConcurrency); - const bgLlm = rateLimitLlmClient(deps.llm, bgLlmSemaphore); - const bgReflectLlm = rateLimitLlmClient(deps.reflectLlm, bgLlmSemaphore); - const bgL3Llm = rateLimitLlmClient(deps.l3Llm ?? deps.llm, bgLlmSemaphore); + const bgLlm = rateLimitLlmClient(deps.llm, bgLlmSemaphore, resources); + const bgReflectLlm = rateLimitLlmClient(deps.reflectLlm, bgLlmSemaphore, resources); + const bgL3Llm = rateLimitLlmClient(deps.l3Llm ?? deps.llm, bgLlmSemaphore, resources); + const bgEmbedder = resources + ? prioritizeEmbedder(deps.embedder, resources, "background") + : deps.embedder; const lightweightMode = algorithm.lightweightMemory.enabled; const captureRunner = createCaptureRunner({ tracesRepo: deps.repos.traces, embeddingRetryQueue: deps.repos.embeddingRetryQueue, episodesRepo: adaptEpisodesRepo(deps.repos.episodes), - embedder: deps.embedder, + embedder: bgEmbedder, llm: bgLlm, // Issue #2148: capture batch reflection emits JSON, so it must use // the main model rather than the potentially thinking-enabled @@ -327,7 +335,7 @@ export function buildPipelineSubscribers( const skillHandle = attachSkillSubscriber({ repos: deps.repos, - embedder: deps.embedder, + embedder: bgEmbedder, llm: bgLlm, bus: buses.skill, l2Bus: buses.l2, @@ -339,7 +347,7 @@ export function buildPipelineSubscribers( const feedbackHandle = attachFeedbackSubscriber({ repos: deps.repos, llm: bgLlm, - embedder: deps.embedder, + embedder: bgEmbedder, bus: buses.feedback, log: log.child({ channel: "core.feedback" }), config: algorithm.feedback, @@ -404,14 +412,17 @@ export function buildPipelineSession( export function buildRetrievalDeps( deps: PipelineDeps, algorithm: PipelineAlgorithmConfig, + resources?: ForegroundResources, ): RetrievalDeps { - const embedder = deps.embedder; + const embedder = resources + ? prioritizeEmbedder(deps.embedder, resources, "foreground") + : deps.embedder; return { repos: wrapRetrievalRepos(deps.repos, deps.namespace), embedder: embedder ? { - embed: (text, role) => - embedder.embedOne({ text, role: role ?? "query" }), + embed: (text, role, options) => + embedder.embedOne({ text, role: role ?? "query" }, options), } : { // Degraded mode: empty vector so vector-scoring falls back to diff --git a/apps/memos-local-plugin/core/pipeline/orchestrator.ts b/apps/memos-local-plugin/core/pipeline/orchestrator.ts index ac48133ad..4ed0427d1 100644 --- a/apps/memos-local-plugin/core/pipeline/orchestrator.ts +++ b/apps/memos-local-plugin/core/pipeline/orchestrator.ts @@ -83,29 +83,39 @@ import { onBroadcastLog } from "../logger/transports/sse-broadcast.js"; import { createEmbeddingRetryWorker, systemErrorEvent } from "../embedding/index.js"; import type { EpisodeSnapshot } from "../session/index.js"; import type { IntentDecision, RelationDecision, TurnRelation } from "../session/types.js"; +import { + createForegroundResources, + prioritizeEmbedder, +} from "../util/foreground-resources.js"; +import { createRequestDeadline } from "../util/request-deadline.js"; function classifyWithTimeout( classifyFn: () => Promise, timeoutMs: number, log: Logger, ): Promise { + let timer: ReturnType | null = null; return Promise.race([ classifyFn(), - new Promise((_, reject) => - setTimeout(() => reject(new Error("classify_timeout")), timeoutMs), - ), - ]).catch((err) => { - log.warn("relation.classify_timeout", { - timeoutMs, - err: err instanceof Error ? err.message : String(err), + new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error("classify_timeout")), timeoutMs); + }), + ]) + .catch((err) => { + log.warn("relation.classify_timeout", { + timeoutMs, + err: err instanceof Error ? err.message : String(err), + }); + return { + relation: "follow_up" as const, + confidence: 0, + reason: "classify_timeout", + signals: ["classify_timeout"], + }; + }) + .finally(() => { + if (timer) clearTimeout(timer); }); - return { - relation: "new_task" as const, - confidence: 0, - reason: "classify_timeout", - signals: ["classify_timeout"], - }; - }); } // ─── Factory ────────────────────────────────────────────────────────────── @@ -115,6 +125,12 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { const algorithm = extractAlgorithmConfig(deps); const lightweightMode = algorithm.lightweightMemory.enabled; const buses = buildPipelineBuses(); + const foregroundResources = createForegroundResources(); + const backgroundEmbedder = prioritizeEmbedder( + deps.embedder, + foregroundResources, + "background", + ); // Session + intent. const session = buildPipelineSession(deps, buses.session); @@ -123,7 +139,13 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { // Pass `session` so the reward runner's `getEpisodeSnapshot` hook // can resolve the live, in-memory episode (with turns populated) // rather than falling back to the empty row from SQLite. - const subs = buildPipelineSubscribers(deps, buses, algorithm, session); + const subs = buildPipelineSubscribers( + deps, + buses, + algorithm, + session, + foregroundResources, + ); // Core-event aggregator. Every internal bus funnels into one stream. const eventListeners = new Set<(e: CoreEvent) => void>(); @@ -160,7 +182,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { let retryEventSeq = 1_000_000; const embeddingRetryWorker = createEmbeddingRetryWorker({ repos: deps.repos, - embedder: deps.embedder, + embedder: backgroundEmbedder, log: log.child({ channel: "core.embedding.retry" }), now: deps.now, onSystemError: (payload, correlationId) => { @@ -366,6 +388,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { userText: string, meta: Record, turnTs?: number, + signal?: AbortSignal, ): Promise { const currentEpId = openEpisodeBySession.get(sessionId); if (currentEpId) { @@ -387,6 +410,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { userMessage: userText, ts: turnTs, meta: lightweightEpisodeMeta(meta), + signal, }); openEpisodeBySession.set(sessionId, snap.id as EpisodeId); return snap; @@ -433,13 +457,20 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { userText: string, meta: Record, agent: AgentKind, + signal?: AbortSignal, ): Promise<{ episode: EpisodeSnapshot; sessionId: SessionId; relation?: string }> { const mergeMode = algorithm.session.followUpMode === "merge_follow_ups"; const mergeCapMs = algorithm.session.mergeMaxGapMs; const turnTs = timestampFromMeta(meta, "startedAtTurnTs"); if (lightweightMode) { - const snap = await startLightweightEpisode(sessionId, userText, meta, turnTs); + const snap = await startLightweightEpisode( + sessionId, + userText, + meta, + turnTs, + signal, + ); return { episode: snap, sessionId, relation: "lightweight_memory" }; } @@ -465,6 +496,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { newUserText: userText, gapMs, prevEpisodeId: currentEpId, + signal, }), algorithm.session.classifyTimeoutMs, log, @@ -582,6 +614,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { userMessage: userText, ts: turnTs, meta: { ...meta, relation: "new_task" }, + signal, }); openEpisodeBySession.set(sessionId, snap.id as EpisodeId); return { episode: snap, sessionId, relation: decision.relation }; @@ -601,6 +634,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { userMessage: userText, ts: turnTs, meta: { ...meta, relation: decision.relation, gapMs }, + signal, }); openEpisodeBySession.set(sessionId, fresh.id as EpisodeId); return { episode: fresh, sessionId, relation: decision.relation }; @@ -645,6 +679,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { newUserText: userText, gapMs, prevEpisodeId: snapshot.id as EpisodeId, + signal, }), algorithm.session.classifyTimeoutMs, log, @@ -748,6 +783,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { userMessage: userText, ts: turnTs, meta, + signal, }); openEpisodeBySession.set(sessionId, snap.id as EpisodeId); return { episode: snap, sessionId, relation: "bootstrap" }; @@ -762,6 +798,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { newUserText: userText, gapMs, prevEpisodeId: prev.episodeId, + signal, }), algorithm.session.classifyTimeoutMs, log, @@ -861,6 +898,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { userMessage: userText, ts: turnTs, meta: { ...meta, relation: "new_task" }, + signal, }); openEpisodeBySession.set(sessionId, snap.id as EpisodeId); return { episode: snap, sessionId, relation: decision.relation }; @@ -871,6 +909,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { userMessage: userText, ts: turnTs, meta: { ...meta, relation: decision.relation }, + signal, }); openEpisodeBySession.set(sessionId, snap.id as EpisodeId); return { episode: snap, sessionId, relation: decision.relation }; @@ -1043,7 +1082,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { // ─── Retrieval entry points ───────────────────────────────────────────── - const retrievalDeps = buildRetrievalDeps(deps, algorithm); + const retrievalDeps = buildRetrievalDeps(deps, algorithm, foregroundResources); const turnStartRetrievalStats = new Map(); function retrievalDepsFor(namespace = deps.namespace): typeof retrievalDeps { @@ -1057,6 +1096,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { async function retrieveTurnStart( input: TurnInputDTO, plan?: RetrievePlan, + signal?: AbortSignal, ): Promise { const ctx = { reason: "turn_start" as const, @@ -1073,6 +1113,8 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { { events: buses.retrieval, skipLlmFilter: input.contextHints?.__memosDeferLlmFilterToCaller === true, + signal, + deadlineAt: input.deadlineAt, plan: plan ? { scenarioId: plan.scenarioId, @@ -1175,13 +1217,45 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { } async function onTurnStartOnce(input: TurnInputDTO): Promise { + const leaveForeground = foregroundResources.enterForeground(); + const deadline = + input.deadlineAt === undefined + ? null + : createRequestDeadline(input.deadlineAt); + const startedAt = Date.now(); + let stage = "ensure_session"; + try { + return await onTurnStartForeground(input, deadline?.signal, (next) => { + stage = next; + }); + } finally { + if (deadline?.signal.aborted) { + log.warn("turn.start.deadline_exceeded", { + sessionId: input.sessionId, + deadlineAt: input.deadlineAt, + elapsedMs: Date.now() - startedAt, + stage, + }); + } + deadline?.dispose(); + leaveForeground(); + } + } + + async function onTurnStartForeground( + input: TurnInputDTO, + signal?: AbortSignal, + setStage: (stage: string) => void = () => {}, + ): Promise { const t0 = now(); + setStage("ensure_session"); const initialSessionId = await ensureSession( input.agent, input.sessionId, input.contextHints, ); + setStage("relation_and_episode"); const routing = await openEpisodeIfNeeded( initialSessionId, input.userText, @@ -1192,6 +1266,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { startedAtTurnTs: input.ts, }, input.agent, + signal, ); const sessionId = routing.sessionId; @@ -1203,10 +1278,12 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { sessionId, episodeId: episode.id as EpisodeId, }; + setStage("intent"); const schedulerIntent = await intentForCurrentTurn({ episode, userText: input.userText, ts: input.ts, + signal, }); const retrievePlan = scheduleInjection({ userText: input.userText, @@ -1240,10 +1317,13 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { retrievalTotalMs: 0, elapsedMs: now() - t0, }); + setStage("complete"); return packet; } - const packet = await retrieveTurnStart(normalized, retrievePlan); + setStage("retrieval"); + const packet = await retrieveTurnStart(normalized, retrievePlan, signal); + setStage("complete"); // Always stamp the routed sessionId + episodeId on the packet so // adapters can correlate the subsequent `agent_end` / `turn.end` // call without needing a separate round-trip to the session @@ -1502,12 +1582,28 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { async function shutdown(reason: string = "shutdown"): Promise { log.info("pipeline.shutdown.begin", { reason }); + // Stop admitting retry jobs, but preserve a bounded grace period for raw + // capture and downstream enrichment. Hermes' bridge owns a 20s outer + // shutdown ceiling, so abort before that rather than either hanging or + // discarding every single-shot session's enrichment immediately. + embeddingRetryWorker.stop(); + const flushPromise = flush(); try { - await flush(); + const completed = await settlesWithin(flushPromise, 15_000); + if (!completed) { + log.warn("pipeline.flush_timeout", { reason, timeoutMs: 15_000 }); + foregroundResources.shutdown(reason); + const aborted = await settlesWithin(flushPromise, 4_000); + if (!aborted) { + log.warn("pipeline.flush_abandoned", { reason, abortWaitMs: 4_000 }); + } + } } catch (err) { log.warn("pipeline.flush_failed", { err: err instanceof Error ? err.message : String(err), }); + } finally { + foregroundResources.shutdown(reason); } // Detach subscribers — prevents late events from re-queuing work. subs.subscriptions.capture.stop(); @@ -1516,13 +1612,26 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { subs.l3.detach(); subs.skills.dispose(); subs.feedback.dispose(); - embeddingRetryWorker.stop(); bridge.dispose(); logSubscription(); session.sessionManager.shutdown(reason); log.info("pipeline.shutdown.done", { reason }); } + async function settlesWithin(promise: Promise, timeoutMs: number): Promise { + let timer: ReturnType | null = null; + try { + return await Promise.race([ + promise.then(() => true), + new Promise((resolve) => { + timer = setTimeout(() => resolve(false), timeoutMs); + }), + ]); + } finally { + if (timer) clearTimeout(timer); + } + } + function now(): number { return (deps.now ?? Date.now)(); } @@ -1589,6 +1698,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { episode: EpisodeSnapshot; userText: string; ts?: number; + signal?: AbortSignal; }): Promise { const firstTurn = input.episode.turns[0]; const isFreshEpisodeForThisTurn = @@ -1603,6 +1713,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { return session.intent.classify(input.userText, { episodeId: input.episode.id as EpisodeId, + signal: input.signal, }); } diff --git a/apps/memos-local-plugin/core/retrieval/llm-filter.ts b/apps/memos-local-plugin/core/retrieval/llm-filter.ts index 142bd626d..87dc2520e 100644 --- a/apps/memos-local-plugin/core/retrieval/llm-filter.ts +++ b/apps/memos-local-plugin/core/retrieval/llm-filter.ts @@ -50,6 +50,7 @@ export interface FilterDeps { llm: LlmClient | null; log: Logger; timeoutMs?: number; + signal?: AbortSignal; config: Pick< RetrievalConfig, | "llmFilterEnabled" @@ -117,6 +118,10 @@ export async function llmFilterCandidates( if (!deps.llm) { return passthrough(ranked, "no_llm"); } + if (deps.signal?.aborted) { + deps.log.debug("llm_filter.deadline_exceeded", { candidateCount: ranked.length }); + return safeCutoff(ranked, deps); + } const bodyChars = deps.config.llmFilterCandidateBodyChars ?? DEFAULT_CANDIDATE_BODY_CHARS; @@ -148,6 +153,7 @@ ${list}`, episodeId: input.episodeId, temperature: 0, timeoutMs: deps.timeoutMs, + signal: deps.signal, // Output is only ordered indices + one bool, but the list can // legitimately be as long as the ranked candidates. maxTokens: filterOutputTokenBudget(ranked.length), diff --git a/apps/memos-local-plugin/core/retrieval/retrieve.ts b/apps/memos-local-plugin/core/retrieval/retrieve.ts index cec076f62..22a7df107 100644 --- a/apps/memos-local-plugin/core/retrieval/retrieve.ts +++ b/apps/memos-local-plugin/core/retrieval/retrieve.ts @@ -75,6 +75,10 @@ export interface RetrieveOptions { * one unified final LLM filter across all routes. */ skipLlmFilter?: boolean; + /** Shared foreground cancellation signal. */ + signal?: AbortSignal; + /** Absolute request deadline used to cap optional LLM filtering. */ + deadlineAt?: number; } export interface RetrievePlanOverride { @@ -258,22 +262,25 @@ async function runAll( degraded: false, }; const queryVec = compiled.text - ? await deps.embedder.embed(compiled.text, "query").then((vec) => { - embeddingStats.ok = true; - return vec; - }).catch((err) => { - const code = (err as { code?: string })?.code; - const message = err instanceof Error ? err.message : String(err); - embeddingStats.degraded = true; - embeddingStats.errorCode = code; - embeddingStats.errorMessage = message; - log.warn("embed_failed", { - reason: ctx.reason, - code, - err: message, - }); - return null; - }) + ? await deps.embedder + .embed(compiled.text, "query", { signal: opts.signal }) + .then((vec) => { + embeddingStats.ok = true; + return vec; + }) + .catch((err) => { + const code = (err as { code?: string })?.code; + const message = err instanceof Error ? err.message : String(err); + embeddingStats.degraded = true; + embeddingStats.errorCode = code; + embeddingStats.errorMessage = message; + log.warn("embed_failed", { + reason: ctx.reason, + code, + err: message, + }); + return null; + }) : null; // The keyword channels (FTS + pattern) work even without an embedder, @@ -415,6 +422,8 @@ async function runAll( llm: deps.llm ?? null, log, config: deps.config, + signal: opts.signal, + timeoutMs: filterTimeoutMs(opts.deadlineAt), }, ); @@ -472,6 +481,8 @@ async function runAll( llm: deps.llm ?? null, log, config: deps.config, + signal: opts.signal, + timeoutMs: filterTimeoutMs(opts.deadlineAt), }, ); @@ -668,6 +679,11 @@ async function runAll( } } +function filterTimeoutMs(deadlineAt?: number): number | undefined { + if (deadlineAt === undefined) return undefined; + return Math.max(1, Math.min(2_000, deadlineAt - Date.now())); +} + function emptyResult( reason: RetrievalReason, agent: AgentKind, diff --git a/apps/memos-local-plugin/core/retrieval/types.ts b/apps/memos-local-plugin/core/retrieval/types.ts index 8ed24935e..bcc9bb7bb 100644 --- a/apps/memos-local-plugin/core/retrieval/types.ts +++ b/apps/memos-local-plugin/core/retrieval/types.ts @@ -679,7 +679,11 @@ export interface RetrievalRepos { /** Abstract embedder surface consumed by retrieval. Mirrors `Embedder`. */ export interface RetrievalEmbedder { - embed: (text: string, role?: "query" | "document") => Promise; + embed: ( + text: string, + role?: "query" | "document", + options?: { signal?: AbortSignal }, + ) => Promise; } export interface RetrievalDeps { diff --git a/apps/memos-local-plugin/core/session/intent-classifier.ts b/apps/memos-local-plugin/core/session/intent-classifier.ts index 41206184d..a4896e07b 100644 --- a/apps/memos-local-plugin/core/session/intent-classifier.ts +++ b/apps/memos-local-plugin/core/session/intent-classifier.ts @@ -46,6 +46,8 @@ export interface IntentClassifierOptions { export interface IntentClassifyOptions { /** Episode id this classification is being run for, when known. */ episodeId?: EpisodeId; + /** Foreground request cancellation propagated to the provider call. */ + signal?: AbortSignal; } export interface IntentClassifier { @@ -91,7 +93,7 @@ export function createIntentClassifier(opts: IntentClassifierOptions = {}): Inte if (!llmDisabled && llm) { try { const result = await withTimeout( - callLlm(llm, text, options?.episodeId), + callLlm(llm, text, options?.episodeId, timeoutMs, options?.signal), timeoutMs, "intent.llm.timeout", ); @@ -207,6 +209,8 @@ async function callLlm( llm: LlmClient, text: string, episodeId?: EpisodeId, + timeoutMs?: number, + signal?: AbortSignal, ): Promise { const rsp = await llm.completeJson<{ kind: unknown; confidence: unknown; reason: unknown }>( [ @@ -217,6 +221,8 @@ async function callLlm( op: "session.intent.classify", phase: "session", episodeId, + timeoutMs, + signal, schemaHint: `{"kind":"task"|"memory_probe"|"chitchat"|"meta"|"unknown","confidence":0..1,"reason":"..."}`, validate: (v) => { const o = v as Record; diff --git a/apps/memos-local-plugin/core/session/manager.ts b/apps/memos-local-plugin/core/session/manager.ts index 44da570b2..8f0f2e79f 100644 --- a/apps/memos-local-plugin/core/session/manager.ts +++ b/apps/memos-local-plugin/core/session/manager.ts @@ -60,6 +60,8 @@ export interface StartEpisodeInput { /** Adapter-provided event time for the first user turn. */ ts?: EpochMs; meta?: Record; + /** Foreground cancellation propagated to intent classification. */ + signal?: AbortSignal; } export interface SessionManager { @@ -267,6 +269,7 @@ export function createSessionManager(deps: SessionManagerDeps): SessionManager { const episodeId = (input.id ?? ids.episode()) as EpisodeId; const intent = await deps.intentClassifier.classify(input.userMessage, { episodeId, + signal: input.signal, }); // Wrap the write+emit in a log context so downstream listeners inherit diff --git a/apps/memos-local-plugin/core/session/relation-classifier.ts b/apps/memos-local-plugin/core/session/relation-classifier.ts index bac4679e0..f0e1e2a24 100644 --- a/apps/memos-local-plugin/core/session/relation-classifier.ts +++ b/apps/memos-local-plugin/core/session/relation-classifier.ts @@ -306,7 +306,11 @@ export function createRelationClassifier( // Step 2: LLM classification. if (!llmDisabled && opts.llm) { try { - const result = await withTimeout(callLlm(opts.llm, input), timeoutMs, "relation.llm.timeout"); + const result = await withTimeout( + callLlm(opts.llm, input, timeoutMs), + timeoutMs, + "relation.llm.timeout", + ); log.debug("llm.ok", { relation: result.relation, confidence: result.confidence, @@ -326,7 +330,7 @@ export function createRelationClassifier( }); try { const arb = await withTimeout( - callArbitration(opts.llm, input), + callArbitration(opts.llm, input, timeoutMs), timeoutMs, "relation.arbitration.timeout", ); @@ -516,7 +520,11 @@ function buildLlmUserContent(input: RelationInput): string { return parts.join("\n\n"); } -async function callLlm(llm: LlmClient, input: RelationInput): Promise { +async function callLlm( + llm: LlmClient, + input: RelationInput, + timeoutMs?: number, +): Promise { const userContent = buildLlmUserContent(input); const rsp = await llm.completeJson<{ relation: unknown; confidence: unknown; reason: unknown }>( @@ -528,6 +536,8 @@ async function callLlm(llm: LlmClient, input: RelationInput): Promise { const o = v as Record; @@ -581,7 +591,11 @@ When in doubt, choose follow_up. Reply JSON ONLY: {"relation":"follow_up"|"new_task","reason":"..."}`; -async function callArbitration(llm: LlmClient, input: RelationInput): Promise { +async function callArbitration( + llm: LlmClient, + input: RelationInput, + timeoutMs?: number, +): Promise { const userContent = [ `CURRENT TASK CONTEXT:\n${(input.prevUserText ?? "").slice(0, 600)}`, `ASSISTANT REPLY:\n${(input.prevAssistantText ?? "").slice(0, 800)}`, @@ -597,6 +611,8 @@ async function callArbitration(llm: LlmClient, input: RelationInput): Promise { const o = v as Record; diff --git a/apps/memos-local-plugin/core/session/types.ts b/apps/memos-local-plugin/core/session/types.ts index 3d34438ea..62ea53f97 100644 --- a/apps/memos-local-plugin/core/session/types.ts +++ b/apps/memos-local-plugin/core/session/types.ts @@ -193,6 +193,8 @@ export interface RelationInput { * is "scoring whether to terminate prevEpisodeId". */ prevEpisodeId?: EpisodeId; + /** Foreground request cancellation propagated to LLM classification. */ + signal?: AbortSignal; } // ─── Event bus ────────────────────────────────────────────────────────────── diff --git a/apps/memos-local-plugin/core/util/foreground-resources.ts b/apps/memos-local-plugin/core/util/foreground-resources.ts new file mode 100644 index 000000000..d818d189f --- /dev/null +++ b/apps/memos-local-plugin/core/util/foreground-resources.ts @@ -0,0 +1,274 @@ +import type { + EmbedCallOptions, + Embedder, + EmbedInput, +} from "../embedding/types.js"; +import type { EmbeddingVector } from "../types.js"; + +export type ResourcePriority = "foreground" | "background"; + +export interface ForegroundResources { + readonly shutdownSignal: AbortSignal; + /** Combine a request signal with the pipeline lifecycle signal. */ + signalFor(signal?: AbortSignal): AbortSignal; + /** Mark the complete turn.start path as foreground work. Idempotent release. */ + enterForeground(): () => void; + /** Background LLM work waits here before acquiring its existing semaphore. */ + waitForBackground(signal?: AbortSignal): Promise; + /** Priority-aware, non-preemptive embedding admission. */ + acquireEmbedding( + priority: ResourcePriority, + signal?: AbortSignal, + ): Promise<() => void>; + /** Reject queued work and cancel provider calls before pipeline drain. */ + shutdown(reason?: string): void; +} + +export interface ForegroundResourceOptions { + embeddingConcurrency?: number; + /** Prevent background starvation during a sustained foreground stream. */ + maxForegroundBurst?: number; +} + +interface Waiter { + resolve: (release: () => void) => void; + reject: (error: Error) => void; + signal?: AbortSignal; + onAbort?: () => void; +} + +interface BackgroundWaiter { + resolve: () => void; + reject: (error: Error) => void; + signal?: AbortSignal; + onAbort?: () => void; +} + +export function createForegroundResources( + options: ForegroundResourceOptions = {}, +): ForegroundResources { + const capacity = Math.max(1, Math.floor(options.embeddingConcurrency ?? 1)); + const maxForegroundBurst = Math.max( + 1, + Math.floor(options.maxForegroundBurst ?? 8), + ); + const embeddingWaiters: Record = { + foreground: [], + background: [], + }; + const backgroundWaiters: BackgroundWaiter[] = []; + let embeddingInUse = 0; + let foregroundActive = 0; + let foregroundBurst = 0; + const shutdownController = new AbortController(); + + function signalFor(signal?: AbortSignal): AbortSignal { + return signal + ? AbortSignal.any([signal, shutdownController.signal]) + : shutdownController.signal; + } + + function abortError(signal?: AbortSignal): Error { + return signal?.reason instanceof Error + ? signal.reason + : new DOMException("resource wait aborted", "AbortError"); + } + + function removeAbortListener(waiter: Waiter | BackgroundWaiter): void { + if (waiter.signal && waiter.onAbort) { + waiter.signal.removeEventListener("abort", waiter.onAbort); + } + } + + function nextEmbeddingWaiter(): { + priority: ResourcePriority; + waiter: Waiter; + } | null { + const foreground = embeddingWaiters.foreground; + const background = embeddingWaiters.background; + if ( + background.length > 0 && + (foreground.length === 0 || foregroundBurst >= maxForegroundBurst) + ) { + return { priority: "background", waiter: background.shift()! }; + } + if (foreground.length > 0) { + return { priority: "foreground", waiter: foreground.shift()! }; + } + if (background.length > 0) { + return { priority: "background", waiter: background.shift()! }; + } + return null; + } + + function drainEmbedding(): void { + while (embeddingInUse < capacity) { + const next = nextEmbeddingWaiter(); + if (!next) return; + removeAbortListener(next.waiter); + embeddingInUse++; + foregroundBurst = next.priority === "foreground" ? foregroundBurst + 1 : 0; + next.waiter.resolve(makeEmbeddingRelease()); + } + } + + function makeEmbeddingRelease(): () => void { + let released = false; + return (): void => { + if (released) return; + released = true; + embeddingInUse--; + drainEmbedding(); + }; + } + + function acquireEmbedding( + priority: ResourcePriority, + signal?: AbortSignal, + ): Promise<() => void> { + signal = signalFor(signal); + if (signal.aborted) return Promise.reject(abortError(signal)); + return new Promise((resolve, reject) => { + const waiter: Waiter = { resolve, reject, signal }; + if (signal) { + waiter.onAbort = () => { + const queue = embeddingWaiters[priority]; + const index = queue.indexOf(waiter); + if (index >= 0) queue.splice(index, 1); + reject(abortError(signal)); + }; + signal.addEventListener("abort", waiter.onAbort, { once: true }); + } + embeddingWaiters[priority].push(waiter); + drainEmbedding(); + }); + } + + function drainBackgroundGate(): void { + if (foregroundActive > 0) return; + for (const waiter of backgroundWaiters.splice(0)) { + removeAbortListener(waiter); + waiter.resolve(); + } + } + + function enterForeground(): () => void { + foregroundActive++; + let left = false; + return (): void => { + if (left) return; + left = true; + foregroundActive--; + drainBackgroundGate(); + }; + } + + function waitForBackground(signal?: AbortSignal): Promise { + signal = signalFor(signal); + if (signal.aborted) return Promise.reject(abortError(signal)); + if (foregroundActive === 0) return Promise.resolve(); + return new Promise((resolve, reject) => { + const waiter: BackgroundWaiter = { resolve, reject, signal }; + if (signal) { + waiter.onAbort = () => { + const index = backgroundWaiters.indexOf(waiter); + if (index >= 0) backgroundWaiters.splice(index, 1); + reject(abortError(signal)); + }; + signal.addEventListener("abort", waiter.onAbort, { once: true }); + } + backgroundWaiters.push(waiter); + }); + } + + function shutdown(reason = "pipeline shutdown"): void { + if (shutdownController.signal.aborted) return; + shutdownController.abort(new DOMException(reason, "AbortError")); + } + + return { + shutdownSignal: shutdownController.signal, + signalFor, + enterForeground, + waitForBackground, + acquireEmbedding, + shutdown, + }; +} + +/** + * Keep the Embedder contract intact while moving provider round-trips behind + * the shared priority arbiter. Background batches are deliberately chunked + * so one enrichment pass cannot monopolize the provider for an entire queue. + */ +export function prioritizeEmbedder( + inner: Embedder | null, + resources: ForegroundResources, + priority: ResourcePriority, + backgroundChunkSize = 8, +): Embedder | null { + if (!inner) return null; + + async function embedOne( + input: string | EmbedInput, + options?: EmbedCallOptions, + ): Promise { + const signal = resources.signalFor(options?.signal); + const callOptions = { ...options, signal }; + if (priority === "background") await resources.waitForBackground(signal); + const release = await resources.acquireEmbedding(priority, signal); + try { + return await inner!.embedOne(input, callOptions); + } finally { + release(); + } + } + + async function embedMany( + inputs: Array, + options?: EmbedCallOptions, + ): Promise { + const signal = resources.signalFor(options?.signal); + const callOptions = { ...options, signal }; + if (priority === "foreground" || inputs.length <= backgroundChunkSize) { + if (priority === "background") await resources.waitForBackground(signal); + const release = await resources.acquireEmbedding(priority, signal); + try { + return await inner!.embedMany(inputs, callOptions); + } finally { + release(); + } + } + + const results: EmbeddingVector[] = []; + for (let start = 0; start < inputs.length; start += backgroundChunkSize) { + await resources.waitForBackground(signal); + const release = await resources.acquireEmbedding(priority, signal); + try { + results.push( + ...await inner!.embedMany(inputs.slice(start, start + backgroundChunkSize), callOptions), + ); + } finally { + release(); + } + } + return results; + } + + return { + get dimensions() { + return inner.dimensions; + }, + get provider() { + return inner.provider; + }, + get model() { + return inner.model; + }, + embedOne, + embedMany, + stats: () => inner.stats(), + resetCache: () => inner.resetCache(), + close: () => inner.close(), + }; +} diff --git a/apps/memos-local-plugin/core/util/rate-limited-llm.ts b/apps/memos-local-plugin/core/util/rate-limited-llm.ts index 4d229b4c9..5bcb863bb 100644 --- a/apps/memos-local-plugin/core/util/rate-limited-llm.ts +++ b/apps/memos-local-plugin/core/util/rate-limited-llm.ts @@ -10,20 +10,26 @@ import type { LlmStreamChunk, } from "../llm/types.js"; import type { Semaphore } from "./semaphore.js"; +import type { ForegroundResources } from "./foreground-resources.js"; /** * Wrap an LLM client so expensive background subscribers share one * process-wide concurrency budget without changing call-site semantics. */ -export function rateLimitLlmClient(client: LlmClient | null, semaphore: Semaphore): LlmClient | null { +export function rateLimitLlmClient( + client: LlmClient | null, + semaphore: Semaphore, + resources?: ForegroundResources, +): LlmClient | null { if (!client) return null; - return new RateLimitedLlmClient(client, semaphore); + return new RateLimitedLlmClient(client, semaphore, resources); } class RateLimitedLlmClient implements LlmClient { constructor( private readonly inner: LlmClient, private readonly semaphore: Semaphore, + private readonly resources?: ForegroundResources, ) {} get provider(): LlmProviderName { @@ -42,9 +48,12 @@ class RateLimitedLlmClient implements LlmClient { messages: LlmMessage[] | string, opts?: LlmCallOptions, ): Promise { - const release = await this.semaphore.acquire(); + const signal = this.resources?.signalFor(opts?.signal) ?? opts?.signal; + const callOpts = signal ? { ...opts, signal } : opts; + await this.resources?.waitForBackground(signal); + const release = await this.semaphore.acquire(signal); try { - return await this.inner.complete(messages, opts); + return await this.inner.complete(messages, callOpts); } finally { release(); } @@ -54,9 +63,12 @@ class RateLimitedLlmClient implements LlmClient { messages: LlmMessage[] | string, opts?: LlmCompleteJsonOptions, ): Promise> { - const release = await this.semaphore.acquire(); + const signal = this.resources?.signalFor(opts?.signal) ?? opts?.signal; + const callOpts = signal ? { ...opts, signal } : opts; + await this.resources?.waitForBackground(signal); + const release = await this.semaphore.acquire(signal); try { - return await this.inner.completeJson(messages, opts); + return await this.inner.completeJson(messages, callOpts); } finally { release(); } @@ -66,9 +78,12 @@ class RateLimitedLlmClient implements LlmClient { messages: LlmMessage[] | string, opts?: LlmCallOptions, ): AsyncIterable { - const release = await this.semaphore.acquire(); + const signal = this.resources?.signalFor(opts?.signal) ?? opts?.signal; + const callOpts = signal ? { ...opts, signal } : opts; + await this.resources?.waitForBackground(signal); + const release = await this.semaphore.acquire(signal); try { - yield* this.inner.stream(messages, opts); + yield* this.inner.stream(messages, callOpts); } finally { release(); } diff --git a/apps/memos-local-plugin/core/util/request-deadline.ts b/apps/memos-local-plugin/core/util/request-deadline.ts new file mode 100644 index 000000000..a7fb816a0 --- /dev/null +++ b/apps/memos-local-plugin/core/util/request-deadline.ts @@ -0,0 +1,37 @@ +export interface RequestDeadline { + readonly signal: AbortSignal; + remainingMs(): number; + dispose(): void; +} + +/** + * Convert an adapter-provided absolute epoch deadline into one abort signal. + * The absolute form survives JSON-RPC transport time and prevents every stage + * from accidentally receiving a fresh timeout budget. + */ +export function createRequestDeadline( + deadlineAt: number, + now: () => number = Date.now, +): RequestDeadline { + const controller = new AbortController(); + const remainingMs = (): number => Math.max(0, deadlineAt - now()); + const initialRemaining = remainingMs(); + let timer: ReturnType | null = null; + + if (!Number.isFinite(deadlineAt) || initialRemaining <= 0) { + controller.abort(new DOMException("request deadline exceeded", "TimeoutError")); + } else { + timer = setTimeout(() => { + controller.abort(new DOMException("request deadline exceeded", "TimeoutError")); + }, initialRemaining); + } + + return { + signal: controller.signal, + remainingMs, + dispose(): void { + if (timer) clearTimeout(timer); + timer = null; + }, + }; +} diff --git a/apps/memos-local-plugin/core/util/retry-after.ts b/apps/memos-local-plugin/core/util/retry-after.ts new file mode 100644 index 000000000..0ca668514 --- /dev/null +++ b/apps/memos-local-plugin/core/util/retry-after.ts @@ -0,0 +1,61 @@ +/** Parse RFC 9110 Retry-After delay-seconds or HTTP-date into milliseconds. */ +export const MAX_RETRY_DELAY_MS = 30_000; + +export function parseRetryAfterMs( + value: string | null | undefined, + nowMs: number = Date.now(), +): number | null { + const raw = value?.trim(); + if (!raw) return null; + if (/^\d+$/.test(raw)) { + const seconds = Number(raw); + const delayMs = seconds * 1_000; + return Number.isSafeInteger(seconds) && Number.isSafeInteger(delayMs) + ? delayMs + : null; + } + // Retry-After only permits IMF-fixdate here. Keeping the shape strict avoids + // JavaScript accepting ambiguous strings such as "1.5" as a legacy date. + if (!/^[A-Za-z]{3}, \d{2} [A-Za-z]{3} \d{4} \d{2}:\d{2}:\d{2} GMT$/.test(raw)) return null; + const at = Date.parse(raw); + if (!Number.isFinite(at)) return null; + return Math.max(0, at - nowMs); +} + +export function retryDelayMs(input: { + attempt: number; + baseMs: number; + jitterMaxMs: number; + retryAfterMs?: number | null; + maxDelayMs?: number; + random?: () => number; +}): number { + const random = input.random ?? Math.random; + const jitter = Math.floor(random() * input.jitterMaxMs); + const exponential = input.baseMs * 2 ** Math.max(0, input.attempt - 1) + jitter; + const requested = Math.max(exponential, input.retryAfterMs ?? 0); + return Math.min(requested, input.maxDelayMs ?? MAX_RETRY_DELAY_MS); +} + +/** Abortable retry wait so request cancellation and shutdown do not leave sleepers behind. */ +export function waitForRetry(delayMs: number, signal?: AbortSignal): Promise { + if (signal?.aborted) return Promise.reject(abortReason(signal)); + if (delayMs <= 0) return Promise.resolve(); + + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + signal?.removeEventListener("abort", onAbort); + resolve(); + }, delayMs); + const onAbort = () => { + clearTimeout(timer); + signal?.removeEventListener("abort", onAbort); + reject(signal ? abortReason(signal) : new DOMException("Aborted", "AbortError")); + }; + signal?.addEventListener("abort", onAbort, { once: true }); + }); +} + +function abortReason(signal: AbortSignal): unknown { + return signal.reason ?? new DOMException("Aborted", "AbortError"); +} diff --git a/apps/memos-local-plugin/core/util/semaphore.ts b/apps/memos-local-plugin/core/util/semaphore.ts index 8dd9b75fc..037db2607 100644 --- a/apps/memos-local-plugin/core/util/semaphore.ts +++ b/apps/memos-local-plugin/core/util/semaphore.ts @@ -1,23 +1,37 @@ export interface Semaphore { - acquire(): Promise<() => void>; + acquire(signal?: AbortSignal): Promise<() => void>; +} + +interface Waiter { + resolve: (release: () => void) => void; + reject: (error: Error) => void; + signal?: AbortSignal; + onAbort?: () => void; } export function createSemaphore(max: number): Semaphore { const limit = Math.max(1, Math.floor(max)); let current = 0; - const waiters: Array<() => void> = []; + const waiters: Waiter[] = []; return { - async acquire() { + async acquire(signal?: AbortSignal) { + if (signal?.aborted) throw abortError(signal); if (current < limit) { current++; return release; } - return new Promise<() => void>((resolve) => { - waiters.push(() => { - current++; - resolve(release); - }); + return new Promise<() => void>((resolve, reject) => { + const waiter: Waiter = { resolve, reject, signal }; + if (signal) { + waiter.onAbort = () => { + const index = waiters.indexOf(waiter); + if (index >= 0) waiters.splice(index, 1); + reject(abortError(signal)); + }; + signal.addEventListener("abort", waiter.onAbort, { once: true }); + } + waiters.push(waiter); }); }, }; @@ -25,6 +39,17 @@ export function createSemaphore(max: number): Semaphore { function release() { current = Math.max(0, current - 1); const next = waiters.shift(); - if (next) next(); + if (!next) return; + if (next.signal && next.onAbort) { + next.signal.removeEventListener("abort", next.onAbort); + } + current++; + next.resolve(release); } } + +function abortError(signal: AbortSignal): Error { + return signal.reason instanceof Error + ? signal.reason + : new DOMException("semaphore wait aborted", "AbortError"); +} diff --git a/apps/memos-local-plugin/tests/python/test_bridge_client.py b/apps/memos-local-plugin/tests/python/test_bridge_client.py index 82763fcea..54acfcd8d 100644 --- a/apps/memos-local-plugin/tests/python/test_bridge_client.py +++ b/apps/memos-local-plugin/tests/python/test_bridge_client.py @@ -1111,7 +1111,7 @@ def test_sync_turn_uses_long_rpc_timeout_for_turn_end(self) -> None: "sessions (issue #2028).", ) - def test_prefetch_uses_long_rpc_timeout_for_turn_start(self) -> None: + def test_prefetch_uses_dedicated_foreground_timeout_for_turn_start(self) -> None: p = self._provider_mod.MemTensorProvider() bridge = RecordingBridge() p._bridge = bridge @@ -1122,12 +1122,45 @@ def test_prefetch_uses_long_rpc_timeout_for_turn_start(self) -> None: self.assertIn("turn.start", methods) start_index = methods.index("turn.start") start_kwargs = bridge.call_kwargs[start_index] - self.assertGreaterEqual( + self.assertGreater(start_kwargs.get("timeout", 0.0), 0.0) + self.assertLessEqual( start_kwargs.get("timeout", 0.0), - self._EXPECTED_LONG_TIMEOUT, - "turn.start suffers the same long-tail latency as turn.end and " - "must share the long RPC timeout (issue #2028).", + self._provider_mod._PREFETCH_RPC_TIMEOUT, + "foreground turn.start must finish before the Hermes host deadline; " + "long capture work keeps the separate issue #2028 timeout.", ) + start_payload = bridge.calls[start_index][1] + self.assertIn("deadlineAt", start_payload) + self.assertGreater(start_payload["deadlineAt"], start_payload["ts"]) + + def test_foreground_reconnect_and_retry_share_one_deadline(self) -> None: + class ClosedBridge: + def request(self, *_args, **_kwargs) -> dict: + raise BridgeError("transport_closed", "bridge closed") + + p = self._provider_mod.MemTensorProvider() + p._bridge = ClosedBridge() + recovered = RecordingBridge() + monotonic_now = [100.0] + + def reconnect(_session_id: str, *, timeout: float) -> None: + self.assertLessEqual(timeout, 6.0) + monotonic_now[0] += 4.0 + p._bridge = recovered + + with ( + patch("memos_provider.time.monotonic", side_effect=lambda: monotonic_now[0]), + patch.object(p, "_reconnect_bridge", side_effect=reconnect), + ): + p._bridge_request_with_retry( + "turn.start", + {"sessionId": "s-1"}, + timeout=6.0, + deadline_monotonic=106.0, + ) + + self.assertEqual(recovered.calls[0][0], "turn.start") + self.assertLessEqual(recovered.call_kwargs[0]["timeout"], 2.0) class ViewerDaemonTests(unittest.TestCase): diff --git a/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py b/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py index 6b95ae61b..ecd0e1ae1 100644 --- a/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py +++ b/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py @@ -480,6 +480,75 @@ def test_prefetch_passes_stable_turn_key_to_bridge(self) -> None: turn_start = next(params for method, params in bridge.calls if method == "turn.start") self.assertEqual(turn_start["turnKey"], "turn-key-session:7") + def test_prefetch_uses_a_dedicated_budget_and_forwards_absolute_deadline(self) -> None: + bridge = FakeBridge() + with ( + patch("memos_provider.ensure_bridge_running", return_value=True), + patch("memos_provider.ensure_viewer_daemon", return_value=True), + patch("memos_provider.MemosBridgeClient", return_value=bridge), + patch("memos_provider._PREFETCH_RPC_TIMEOUT", 6.0), + patch("memos_provider.time.time", return_value=1_700_000_000.0), + ): + provider = memos_provider.MemTensorProvider() + provider.initialize("budget-session") + provider.on_turn_start(1, "recall the build decision") + with patch.object( + provider, + "_bridge_request_with_retry", + wraps=provider._bridge_request_with_retry, + ) as request: + provider.prefetch("recall the build decision") + + turn_start = next(params for method, params in bridge.calls if method == "turn.start") + self.assertEqual(turn_start["deadlineAt"], 1_700_000_005_750) + request.assert_called_once() + self.assertLessEqual(request.call_args.kwargs["timeout"], 6.0) + self.assertIn("deadline_monotonic", request.call_args.kwargs) + + def test_prefetch_budget_includes_bridge_ensure_time(self) -> None: + bridge = FakeBridge() + monotonic_now = [100.0] + + def ensure_bridge(_session_id: str, *, timeout: float) -> bool: + self.assertAlmostEqual(timeout, 6.0, places=3) + monotonic_now[0] += 2.5 + return True + + with ( + patch("memos_provider.ensure_bridge_running", return_value=True), + patch("memos_provider.ensure_viewer_daemon", return_value=True), + patch("memos_provider.MemosBridgeClient", return_value=bridge), + patch("memos_provider._PREFETCH_RPC_TIMEOUT", 6.0), + patch("memos_provider.time.time", return_value=1_700_000_000.0), + patch("memos_provider.time.monotonic", side_effect=lambda: monotonic_now[0]), + ): + provider = memos_provider.MemTensorProvider() + provider.initialize("budget-session") + provider.on_turn_start(1, "recall the build decision") + with ( + patch.object(provider, "_ensure_bridge", side_effect=ensure_bridge), + patch.object( + provider, + "_bridge_request_with_retry", + wraps=provider._bridge_request_with_retry, + ) as request, + ): + provider.prefetch("recall the build decision") + + self.assertLessEqual(request.call_args.kwargs["timeout"], 3.5) + turn_start = next(params for method, params in bridge.calls if method == "turn.start") + self.assertEqual(turn_start["deadlineAt"], 1_700_000_005_750) + + def test_prefetch_timeout_config_rejects_non_positive_values(self) -> None: + with patch.dict("os.environ", {"MEMOS_HERMES_PREFETCH_RPC_TIMEOUT": "0"}): + self.assertEqual(memos_provider._prefetch_rpc_timeout_default(), 6.0) + with patch.dict("os.environ", {"MEMOS_HERMES_PREFETCH_RPC_TIMEOUT": "nan"}): + self.assertEqual(memos_provider._prefetch_rpc_timeout_default(), 6.0) + with patch.dict("os.environ", {"MEMOS_HERMES_PREFETCH_RPC_TIMEOUT": "4.5"}): + self.assertEqual(memos_provider._prefetch_rpc_timeout_default(), 4.5) + with patch.dict("os.environ", {"MEMOS_HERMES_PREFETCH_RPC_TIMEOUT": "30"}): + self.assertEqual(memos_provider._prefetch_rpc_timeout_default(), 7.0) + def test_prefetch_suppresses_memory_injection_for_explicit_delegation(self) -> None: bridge = FakeBridge() with ( diff --git a/apps/memos-local-plugin/tests/unit/bridge/methods.test.ts b/apps/memos-local-plugin/tests/unit/bridge/methods.test.ts index 8ee5fa921..fc9f921e1 100644 --- a/apps/memos-local-plugin/tests/unit/bridge/methods.test.ts +++ b/apps/memos-local-plugin/tests/unit/bridge/methods.test.ts @@ -344,6 +344,17 @@ describe("makeDispatcher", () => { ).rejects.toSatisfy( (err) => err instanceof MemosError && err.code === "invalid_argument", ); + await expect( + dispatch("turn.start", { + agent: "openclaw", + sessionId: "s-1", + userText: "hi", + ts: 123, + deadlineAt: "soon", + }), + ).rejects.toSatisfy( + (err) => err instanceof MemosError && err.code === "invalid_argument", + ); }); it("feedback.submit forwards the DTO shape intact", async () => { diff --git a/apps/memos-local-plugin/tests/unit/embedding/embedder.test.ts b/apps/memos-local-plugin/tests/unit/embedding/embedder.test.ts index 3c608a53a..7c0b6bfff 100644 --- a/apps/memos-local-plugin/tests/unit/embedding/embedder.test.ts +++ b/apps/memos-local-plugin/tests/unit/embedding/embedder.test.ts @@ -75,6 +75,23 @@ describe("embedder facade", () => { expect(Array.from(v)).toEqual([3, 97, 0]); // a=97 }); + it("forwards the caller abort signal to the provider", async () => { + const seen: Array = []; + const p: EmbeddingProvider = { + name: "openai_compatible", + async embed(texts, _role, ctx) { + seen.push(ctx.signal); + return texts.map(() => [1, 2, 3]); + }, + }; + const e = createEmbedderWithProvider(cfg(), p); + const controller = new AbortController(); + + await e.embedOne("signal", { signal: controller.signal }); + + expect(seen).toEqual([controller.signal]); + }); + it("dedups identical inputs into one provider call", async () => { const p = new FakeProvider(); const e = createEmbedderWithProvider(cfg(), p); diff --git a/apps/memos-local-plugin/tests/unit/embedding/fetcher.test.ts b/apps/memos-local-plugin/tests/unit/embedding/fetcher.test.ts index 3b9ee5bf2..73f37fed4 100644 --- a/apps/memos-local-plugin/tests/unit/embedding/fetcher.test.ts +++ b/apps/memos-local-plugin/tests/unit/embedding/fetcher.test.ts @@ -21,6 +21,7 @@ describe("embedding/fetcher", () => { vi.useRealTimers(); // retry backoff uses real setTimeout; keep it real but short }); afterEach(() => { + vi.useRealTimers(); vi.unstubAllGlobals(); }); @@ -81,6 +82,32 @@ describe("embedding/fetcher", () => { expect(f).toHaveBeenCalledTimes(2); }); + it("honors Retry-After HTTP-date before retrying a 429", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-08-04T00:00:00.000Z")); + const f = mockFetch([ + new Response("rate limited", { + status: 429, + headers: { "Retry-After": "Tue, 04 Aug 2026 00:00:03 GMT" }, + }), + new Response(JSON.stringify({ ok: 1 }), { status: 200 }), + ]); + + const pending = httpPostJson<{ ok: number }>({ + url: "https://x", + body: {}, + provider: "cohere", + log: nullLogger(), + maxRetries: 1, + }); + await vi.advanceTimersByTimeAsync(2_999); + expect(f).toHaveBeenCalledTimes(1); + await vi.advanceTimersByTimeAsync(1); + await expect(pending).resolves.toEqual({ ok: 1 }); + expect(f).toHaveBeenCalledTimes(2); + vi.useRealTimers(); + }); + it("does not retry on 400", async () => { mockFetch([new Response("bad", { status: 400 })]); await expect( diff --git a/apps/memos-local-plugin/tests/unit/embedding/retry-worker.test.ts b/apps/memos-local-plugin/tests/unit/embedding/retry-worker.test.ts index f08a37f32..5d7cd99b9 100644 --- a/apps/memos-local-plugin/tests/unit/embedding/retry-worker.test.ts +++ b/apps/memos-local-plugin/tests/unit/embedding/retry-worker.test.ts @@ -201,4 +201,30 @@ describe("embedding retry worker", () => { last_error: "embedding retry target not found: trace:tr_missing", }); }); + + it("does not claim new retry jobs after stop during shutdown", async () => { + handle.repos.embeddingRetryQueue.enqueue({ + id: "er_shutdown", + targetKind: "trace", + targetId: "tr_retry", + vectorField: "vec_summary", + sourceText: "do not start during shutdown", + now: NOW, + }); + const worker = createEmbeddingRetryWorker({ + repos: handle.repos, + embedder: fakeEmbedder({ dimensions: 8 }), + log: rootLogger.child({ channel: "test.embedding.retry" }), + now: () => NOW, + }); + + worker.stop(); + await worker.flush(); + + expect(queueRow(handle, "er_shutdown")).toMatchObject({ + status: "pending", + attempts: 0, + claimed_by: null, + }); + }); }); diff --git a/apps/memos-local-plugin/tests/unit/llm/fetcher.test.ts b/apps/memos-local-plugin/tests/unit/llm/fetcher.test.ts index a3941b7aa..8bbc4ab7a 100644 --- a/apps/memos-local-plugin/tests/unit/llm/fetcher.test.ts +++ b/apps/memos-local-plugin/tests/unit/llm/fetcher.test.ts @@ -29,7 +29,78 @@ function mockFetch(replies: Array) { describe("llm/fetcher", () => { beforeAll(() => initTestLogger()); - afterEach(() => vi.unstubAllGlobals()); + afterEach(() => { + vi.useRealTimers(); + vi.unstubAllGlobals(); + }); + + it("honors Retry-After delay-seconds before retrying a 429", async () => { + vi.useFakeTimers(); + const f = mockFetch([ + new Response("slow down", { status: 429, headers: { "Retry-After": "2" } }), + new Response(JSON.stringify({ ok: 1 }), { status: 200 }), + ]); + + const pending = httpPostJson({ + url: "https://x", + body: {}, + timeoutMs: 5_000, + maxRetries: 1, + provider: "openai_compatible", + log: nullLog(), + }); + await vi.advanceTimersByTimeAsync(1_999); + expect(f).toHaveBeenCalledTimes(1); + await vi.advanceTimersByTimeAsync(1); + await expect(pending).resolves.toMatchObject({ json: { ok: 1 } }); + expect(f).toHaveBeenCalledTimes(2); + vi.useRealTimers(); + }); + + it("aborts while waiting for Retry-After", async () => { + vi.useFakeTimers(); + const ctrl = new AbortController(); + const f = mockFetch([ + new Response("slow down", { status: 429, headers: { "Retry-After": "120" } }), + ]); + + const pending = httpPostJson({ + url: "https://x", + body: {}, + timeoutMs: 5_000, + maxRetries: 1, + signal: ctrl.signal, + provider: "openai_compatible", + log: nullLog(), + }); + await vi.advanceTimersByTimeAsync(0); + ctrl.abort(); + await expect(pending).rejects.toBeInstanceOf(MemosError); + expect(f).toHaveBeenCalledTimes(1); + vi.useRealTimers(); + }); + + it("caps a long Retry-After from a 503 response", async () => { + vi.useFakeTimers(); + const f = mockFetch([ + new Response("maintenance", { status: 503, headers: { "Retry-After": "120" } }), + new Response(JSON.stringify({ ok: 1 }), { status: 200 }), + ]); + + const pending = httpPostJson({ + url: "https://x", + body: {}, + timeoutMs: 120_000, + maxRetries: 1, + provider: "openai_compatible", + log: nullLog(), + }); + await vi.advanceTimersByTimeAsync(29_999); + expect(f).toHaveBeenCalledTimes(1); + await vi.advanceTimersByTimeAsync(1); + await expect(pending).resolves.toMatchObject({ json: { ok: 1 } }); + expect(f).toHaveBeenCalledTimes(2); + }); it("returns parsed JSON on 200", async () => { mockFetch([new Response(JSON.stringify({ a: 1 }), { status: 200 })]); diff --git a/apps/memos-local-plugin/tests/unit/pipeline/orchestrator.test.ts b/apps/memos-local-plugin/tests/unit/pipeline/orchestrator.test.ts index a70fdcf6b..1e79c861a 100644 --- a/apps/memos-local-plugin/tests/unit/pipeline/orchestrator.test.ts +++ b/apps/memos-local-plugin/tests/unit/pipeline/orchestrator.test.ts @@ -77,6 +77,39 @@ afterEach(async () => { }); describe("pipeline/orchestrator", () => { + it("degrades retrieval at the adapter deadline and aborts the provider call", async () => { + const base = fakeEmbedder({ dimensions: 384 }); + let sawAbort = false; + const embedder = { + ...base, + async embedOne(input: Parameters[0], options?: { signal?: AbortSignal }) { + return await new Promise>>((resolve, reject) => { + const onAbort = () => { + sawAbort = true; + reject(new DOMException("deadline", "AbortError")); + }; + if (options?.signal?.aborted) return onAbort(); + options?.signal?.addEventListener("abort", onAbort, { once: true }); + void resolve; + }); + }, + }; + pipeline = createPipeline(buildDeps(dbHandle!, embedder)); + const startedAt = Date.now(); + + const packet = await pipeline.onTurnStart({ + agent: "hermes", + sessionId: "s-deadline", + userText: "find the previous build decision", + ts: Date.now(), + deadlineAt: Date.now() + 25, + }); + + expect(sawAbort).toBe(true); + expect(Date.now() - startedAt).toBeLessThan(500); + expect(packet.reason).toBe("turn_start"); + }); + it("threads a dedicated l3Llm through to the handle", () => { const l3Llm = fakeLlm({ completeJson: {} }); pipeline = createPipeline({ ...buildDeps(dbHandle!), l3Llm }); diff --git a/apps/memos-local-plugin/tests/unit/session/intent-classifier.test.ts b/apps/memos-local-plugin/tests/unit/session/intent-classifier.test.ts index 6b37ae55f..c3321635a 100644 --- a/apps/memos-local-plugin/tests/unit/session/intent-classifier.test.ts +++ b/apps/memos-local-plugin/tests/unit/session/intent-classifier.test.ts @@ -109,6 +109,25 @@ describe("session/intent-classifier", () => { expect(d.signals).toEqual(["llm"]); }); + it("forwards the foreground abort signal and classifier timeout", async () => { + let seen: { signal?: AbortSignal; timeoutMs?: number } | undefined; + const llm = fakeLlm(() => ({ kind: "task", confidence: 0.8, reason: "task" })); + const original = llm.completeJson.bind(llm); + llm.completeJson = async (messages, opts) => { + seen = opts; + return original(messages, opts); + }; + const controller = new AbortController(); + const c = createIntentClassifier({ llm, timeoutMs: 321 }); + + await c.classify("investigate an ambiguous pipeline issue", { + signal: controller.signal, + }); + + expect(seen?.signal).toBe(controller.signal); + expect(seen?.timeoutMs).toBe(321); + }); + it("LLM failure falls back to heuristic", async () => { const c = createIntentClassifier({ llm: fakeLlm(() => { diff --git a/apps/memos-local-plugin/tests/unit/session/relation-classifier.test.ts b/apps/memos-local-plugin/tests/unit/session/relation-classifier.test.ts index 575a4b44a..a17d3d9ce 100644 --- a/apps/memos-local-plugin/tests/unit/session/relation-classifier.test.ts +++ b/apps/memos-local-plugin/tests/unit/session/relation-classifier.test.ts @@ -119,6 +119,33 @@ describe("relation-classifier — V7 §0.1", () => { expect(d.llmModel).toBe("fake/test-model"); }); + it("forwards the foreground abort signal and classifier timeout", async () => { + let seen: { signal?: AbortSignal; timeoutMs?: number } | undefined; + const controller = new AbortController(); + const c = createRelationClassifier({ + timeoutMs: 456, + llm: { + completeJson: async (_messages, opts) => { + seen = opts; + return { + value: { relation: "follow_up", confidence: 0.9, reason: "same task" }, + servedBy: "fake/llm", + } as never; + }, + } as LlmClient, + }); + + await c.classify({ + prevUserText: "investigate retrieval latency", + prevAssistantText: "I found several possible causes.", + newUserText: "could the queue contribute to this behavior?", + signal: controller.signal, + }); + + expect(seen?.signal).toBe(controller.signal); + expect(seen?.timeoutMs).toBe(456); + }); + it("falls back to heuristic when LLM throws", async () => { const llm: Partial = { completeJson: async () => { diff --git a/apps/memos-local-plugin/tests/unit/util/foreground-resources.test.ts b/apps/memos-local-plugin/tests/unit/util/foreground-resources.test.ts new file mode 100644 index 000000000..1380b9a47 --- /dev/null +++ b/apps/memos-local-plugin/tests/unit/util/foreground-resources.test.ts @@ -0,0 +1,138 @@ +import { describe, expect, it } from "vitest"; + +import { + createForegroundResources, + prioritizeEmbedder, +} from "../../../core/util/foreground-resources.js"; +import { fakeEmbedder } from "../../helpers/fake-embedder.js"; + +describe("foreground resources", () => { + it("admits a queued foreground embedding before queued background work", async () => { + const resources = createForegroundResources({ embeddingConcurrency: 1 }); + const first = await resources.acquireEmbedding("background"); + const order: string[] = []; + + const background = resources.acquireEmbedding("background").then((release) => { + order.push("background"); + release(); + }); + const foreground = resources.acquireEmbedding("foreground").then((release) => { + order.push("foreground"); + release(); + }); + + first(); + await Promise.all([foreground, background]); + + expect(order).toEqual(["foreground", "background"]); + }); + + it("lets background work progress after a bounded foreground burst", async () => { + const resources = createForegroundResources({ + embeddingConcurrency: 1, + maxForegroundBurst: 2, + }); + const first = await resources.acquireEmbedding("foreground"); + const order: string[] = []; + + const background = resources.acquireEmbedding("background").then((release) => { + order.push("background"); + release(); + }); + const foreground1 = resources.acquireEmbedding("foreground").then((release) => { + order.push("foreground-1"); + release(); + }); + const foreground2 = resources.acquireEmbedding("foreground").then((release) => { + order.push("foreground-2"); + release(); + }); + + first(); + await Promise.all([background, foreground1, foreground2]); + + expect(order).toEqual(["foreground-1", "background", "foreground-2"]); + }); + + it("does not start background work while a foreground turn is active", async () => { + const resources = createForegroundResources(); + const leaveForeground = resources.enterForeground(); + let started = false; + + const waiting = resources.waitForBackground().then(() => { + started = true; + }); + await Promise.resolve(); + expect(started).toBe(false); + + leaveForeground(); + await waiting; + expect(started).toBe(true); + }); + + it("removes an aborted embedding waiter without consuming capacity", async () => { + const resources = createForegroundResources({ embeddingConcurrency: 1 }); + const first = await resources.acquireEmbedding("background"); + const controller = new AbortController(); + const waiting = resources.acquireEmbedding("foreground", controller.signal); + + controller.abort(); + await expect(waiting).rejects.toMatchObject({ name: "AbortError" }); + first(); + + const release = await resources.acquireEmbedding("background"); + release(); + }); + + it("chunks background embedding batches and yields between chunks", async () => { + const resources = createForegroundResources({ embeddingConcurrency: 1 }); + const base = fakeEmbedder({ dimensions: 4 }); + const batchSizes: number[] = []; + const inner = { + ...base, + async embedMany(...args: Parameters) { + batchSizes.push(args[0].length); + return base.embedMany(...args); + }, + }; + const background = prioritizeEmbedder(inner, resources, "background", 2)!; + + await background.embedMany(["a", "b", "c", "d", "e"]); + + expect(batchSizes).toEqual([2, 2, 1]); + }); + + it("aborts queued and in-flight provider work during shutdown", async () => { + const resources = createForegroundResources({ embeddingConcurrency: 1 }); + const base = fakeEmbedder({ dimensions: 4 }); + let providerSignal: AbortSignal | undefined; + const inner = { + ...base, + async embedOne( + _input: Parameters[0], + options?: Parameters[1], + ) { + providerSignal = options?.signal; + return await new Promise((_resolve, reject) => { + if (options?.signal?.aborted) { + reject(options.signal.reason); + return; + } + options?.signal?.addEventListener( + "abort", + () => reject(options.signal?.reason), + { once: true }, + ); + }); + }, + }; + const background = prioritizeEmbedder(inner, resources, "background")!; + const pending = background.embedOne("slow background work"); + await Promise.resolve(); + + resources.shutdown("test shutdown"); + + await expect(pending).rejects.toMatchObject({ name: "AbortError" }); + expect(providerSignal?.aborted).toBe(true); + }); +}); diff --git a/apps/memos-local-plugin/tests/unit/util/request-deadline.test.ts b/apps/memos-local-plugin/tests/unit/util/request-deadline.test.ts new file mode 100644 index 000000000..8d6b8d5a2 --- /dev/null +++ b/apps/memos-local-plugin/tests/unit/util/request-deadline.test.ts @@ -0,0 +1,35 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { createRequestDeadline } from "../../../core/util/request-deadline.js"; + +afterEach(() => { + vi.useRealTimers(); +}); + +describe("createRequestDeadline", () => { + it("aborts at the absolute deadline and reports no remaining budget", async () => { + vi.useFakeTimers(); + vi.setSystemTime(1_000); + + const deadline = createRequestDeadline(1_250); + expect(deadline.remainingMs()).toBe(250); + expect(deadline.signal.aborted).toBe(false); + + await vi.advanceTimersByTimeAsync(250); + + expect(deadline.signal.aborted).toBe(true); + expect(deadline.remainingMs()).toBe(0); + deadline.dispose(); + }); + + it("treats an already-expired deadline as immediately aborted", () => { + vi.useFakeTimers(); + vi.setSystemTime(2_000); + + const deadline = createRequestDeadline(1_999); + + expect(deadline.signal.aborted).toBe(true); + expect(deadline.remainingMs()).toBe(0); + deadline.dispose(); + }); +}); diff --git a/apps/memos-local-plugin/tests/unit/util/retry-after.test.ts b/apps/memos-local-plugin/tests/unit/util/retry-after.test.ts new file mode 100644 index 000000000..0d412939b --- /dev/null +++ b/apps/memos-local-plugin/tests/unit/util/retry-after.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from "vitest"; + +import { parseRetryAfterMs, retryDelayMs } from "../../../core/util/retry-after.js"; + +describe("parseRetryAfterMs", () => { + it("parses delay-seconds", () => { + expect(parseRetryAfterMs("3", 1_000)).toBe(3_000); + expect(parseRetryAfterMs("0", 1_000)).toBe(0); + }); + + it("parses an HTTP-date relative to the supplied clock", () => { + const now = Date.parse("2026-08-04T00:00:00.000Z"); + expect(parseRetryAfterMs("Tue, 04 Aug 2026 00:00:05 GMT", now)).toBe(5_000); + }); + + it("clamps past HTTP-dates and rejects malformed values", () => { + const now = Date.parse("2026-08-04T00:00:00.000Z"); + expect(parseRetryAfterMs("Mon, 03 Aug 2026 23:59:59 GMT", now)).toBe(0); + expect(parseRetryAfterMs("1.5", now)).toBeNull(); + expect(parseRetryAfterMs("9007199254740991", now)).toBeNull(); + expect(parseRetryAfterMs("later", now)).toBeNull(); + expect(parseRetryAfterMs(null, now)).toBeNull(); + }); + + it("caps a provider supplied Retry-After before scheduling a timer", () => { + expect(retryDelayMs({ + attempt: 1, + baseMs: 200, + jitterMaxMs: 0, + retryAfterMs: 120_000, + maxDelayMs: 30_000, + })).toBe(30_000); + }); +}); diff --git a/apps/memos-local-plugin/tests/unit/util/semaphore.test.ts b/apps/memos-local-plugin/tests/unit/util/semaphore.test.ts new file mode 100644 index 000000000..2a5308ad2 --- /dev/null +++ b/apps/memos-local-plugin/tests/unit/util/semaphore.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, it } from "vitest"; + +import { createSemaphore } from "../../../core/util/semaphore.js"; + +describe("semaphore", () => { + it("removes an aborted waiter so shutdown cannot hang behind active work", async () => { + const semaphore = createSemaphore(1); + const release = await semaphore.acquire(); + const controller = new AbortController(); + const waiting = semaphore.acquire(controller.signal); + + controller.abort(); + await expect(waiting).rejects.toMatchObject({ name: "AbortError" }); + release(); + + const next = await semaphore.acquire(); + next(); + }); +}); From a159dfc40da8f5cb797cdbe955dcd782b210c4af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=81=E5=9C=A8=E5=90=B5=E7=9D=80=E5=90=83=E7=B3=96?= Date: Tue, 4 Aug 2026 16:40:49 +0800 Subject: [PATCH 15/18] fix(plugin): unblock Hermes bridge reader --- .../hermes/memos_provider/bridge_client.py | 137 +++++++++++++----- .../tests/python/test_bridge_client.py | 126 ++++++++++++++++ 2 files changed, 227 insertions(+), 36 deletions(-) diff --git a/apps/memos-local-plugin/adapters/hermes/memos_provider/bridge_client.py b/apps/memos-local-plugin/adapters/hermes/memos_provider/bridge_client.py index 6863ba1ef..016c93ae5 100644 --- a/apps/memos-local-plugin/adapters/hermes/memos_provider/bridge_client.py +++ b/apps/memos-local-plugin/adapters/hermes/memos_provider/bridge_client.py @@ -17,6 +17,7 @@ import json import logging import os +import queue import shutil import subprocess import threading @@ -32,6 +33,7 @@ logger = logging.getLogger(__name__) HOST_HANDLER_WAIT_SECONDS = 5.0 +HOST_HANDLER_QUEUE_CAPACITY = 16 # ─── Module-level singleton tracker ───────────────────────────────────── # Each entry maps an ``(agent, no_viewer, runtime_home)`` key to the @@ -152,12 +154,16 @@ def __init__( # Reverse-direction handlers: the bridge can send us a # JSON-RPC request via `serverRequest(...)` (e.g. # `host.llm.complete` for fallback LLM calls). Registered - # methods run on the dedicated reader thread; long-running - # work should spawn its own worker if it needs to. Each - # handler returns a JSON-serialisable value or raises to - # surface a JSON-RPC error back to the bridge. + # methods run on one bounded, daemon worker. Keeping execution + # serial preserves the adapter's previous concurrency contract while + # preventing a slow host LLM call from blocking stdout response + # demultiplexing for every shared provider lease. self._host_handlers: dict[str, Callable[[dict[str, Any]], Any]] = {} self._host_handlers_cv = threading.Condition() + self._host_handler_queue: queue.Queue[ + tuple[Any, str, dict[str, Any]] | None + ] = queue.Queue(maxsize=HOST_HANDLER_QUEUE_CAPACITY) + self._host_handler_stop = threading.Event() self._closed = False plugin_root = Path(__file__).resolve().parent.parent.parent.parent @@ -226,6 +232,12 @@ def __init__( env=env, cwd=str(plugin_root), ) + self._host_handler_worker = threading.Thread( + target=self._host_handler_loop, + daemon=True, + name="memos-bridge-host-handler", + ) + self._host_handler_worker.start() self._reader = threading.Thread( target=self._read_loop, daemon=True, @@ -348,7 +360,7 @@ def notify(self, method: str, params: Any = None) -> None: try: self._proc.stdin.write(payload + "\n") self._proc.stdin.flush() - except (BrokenPipeError, OSError): + except (BrokenPipeError, OSError, ValueError): pass def on_event(self, cb: Callable[[dict[str, Any]], None]) -> None: @@ -365,11 +377,9 @@ def register_host_handler( """Register a handler for bridge → adapter (reverse) requests. The Node-side bridge calls these via ``stdio.serverRequest``. - Most-recent registration wins. The handler runs on the reader - thread; if it blocks for a long time it stalls every other - bridge → adapter notification, so handlers that need to do - heavy work (e.g. an LLM call) are still expected to return - within the bridge-side timeout (default 60 s). + Most-recent registration wins. Handlers run serially on a bounded + daemon worker so a long-running host LLM call cannot stall the reader + thread that resolves unrelated foreground JSON-RPC responses. """ with self._host_handlers_cv: self._host_handlers[method] = handler @@ -381,6 +391,7 @@ def close(self) -> None: with self._host_handlers_cv: self._closed = True self._host_handlers_cv.notify_all() + self._stop_host_handler_worker() # Drop self from the module-level singleton tracker (issue #1910) # BEFORE the potentially-slow stdin/SIGTERM/SIGKILL dance. We @@ -435,6 +446,7 @@ def _abort_pending(self, reason: str) -> None: with self._host_handlers_cv: self._closed = True self._host_handlers_cv.notify_all() + self._stop_host_handler_worker() with self._lock: for entry in list(self._pending.values()): entry["error"] = { @@ -477,7 +489,9 @@ def _read_loop(self) -> None: # Reverse-direction request: the bridge is asking the # adapter to do something (e.g. run a fallback LLM call # via `host.llm.complete`). Dispatch to the registered - # handler and write the response back synchronously. + # handler on the bounded worker. The reader must return to + # stdout immediately so a slow host LLM callback cannot + # head-of-line block normal JSON-RPC responses. method = msg.get("method") rpc_id = msg.get("id") if ( @@ -486,33 +500,10 @@ def _read_loop(self) -> None: and "result" not in msg and "error" not in msg ): - handler = self._host_handler_for(method) - if handler is None: - self._send_response( - rpc_id, - error={ - "code": -32601, - "message": f"method not found: {method}", - "data": {"code": "unknown_method"}, - }, - ) - continue params = msg.get("params") or {} if not isinstance(params, dict): params = {} - try: - result = handler(params) - self._send_response(rpc_id, result=result) - except Exception as err: - logger.warning("host handler %s failed: %s", method, err) - self._send_response( - rpc_id, - error={ - "code": -32000, - "message": str(err) or err.__class__.__name__, - "data": {"code": "host_handler_failed"}, - }, - ) + self._dispatch_host_request(rpc_id, method, params) continue except Exception: # Any unexpected exception in the reader loop still needs @@ -527,6 +518,80 @@ def _read_loop(self) -> None: # instead of waiting for each 30 s per-request timeout. self._abort_pending("bridge subprocess exited") + def _dispatch_host_request( + self, + rpc_id: Any, + method: str, + params: dict[str, Any], + ) -> None: + """Queue reverse RPC work without ever blocking the reader thread.""" + if self._closed: + return + try: + self._host_handler_queue.put_nowait((rpc_id, method, params)) + except queue.Full: + logger.warning("host handler queue full; rejecting %s", method) + self._send_response( + rpc_id, + error={ + "code": -32000, + "message": "host handler queue is full", + "data": {"code": "host_handler_busy"}, + }, + ) + + def _host_handler_loop(self) -> None: + """Run reverse RPC handlers serially away from stdout demultiplexing.""" + while True: + request = self._host_handler_queue.get() + try: + if request is None: + return + rpc_id, method, params = request + if self._closed: + continue + handler = self._host_handler_for(method) + if handler is None: + self._send_response( + rpc_id, + error={ + "code": -32601, + "message": f"method not found: {method}", + "data": {"code": "unknown_method"}, + }, + ) + continue + try: + result = handler(params) + self._send_response(rpc_id, result=result) + except Exception as err: + logger.warning("host handler %s failed: %s", method, err) + self._send_response( + rpc_id, + error={ + "code": -32000, + "message": str(err) or err.__class__.__name__, + "data": {"code": "host_handler_failed"}, + }, + ) + finally: + self._host_handler_queue.task_done() + + def _stop_host_handler_worker(self) -> None: + """Discard queued callbacks and ask the daemon worker to exit.""" + if self._host_handler_stop.is_set(): + return + self._host_handler_stop.set() + while True: + try: + self._host_handler_queue.get_nowait() + except queue.Empty: + break + else: + self._host_handler_queue.task_done() + with contextlib.suppress(queue.Full): + self._host_handler_queue.put_nowait(None) + def _host_handler_for( self, method: str, @@ -568,7 +633,7 @@ def _send_response( try: self._proc.stdin.write(json.dumps(payload, ensure_ascii=False) + "\n") self._proc.stdin.flush() - except (BrokenPipeError, OSError): + except (BrokenPipeError, OSError, ValueError): pass def _stderr_loop(self) -> None: diff --git a/apps/memos-local-plugin/tests/python/test_bridge_client.py b/apps/memos-local-plugin/tests/python/test_bridge_client.py index 54acfcd8d..1bd9dce3f 100644 --- a/apps/memos-local-plugin/tests/python/test_bridge_client.py +++ b/apps/memos-local-plugin/tests/python/test_bridge_client.py @@ -448,6 +448,132 @@ def test_reverse_request_waits_for_late_host_handler_registration(self) -> None: self.assertNotIn("error", response) client.close() + def test_slow_reverse_handler_does_not_block_regular_rpc_responses(self) -> None: + """A host LLM callback must not stall the stdout response demux. + + ``host.llm.complete`` can legitimately spend several seconds in the + Hermes model client. The bridge reader still has to resolve an + unrelated foreground ``turn.start`` response during that + window; otherwise one background callback head-of-line blocks every + provider lease sharing the process. + """ + client = MemosBridgeClient(bridge_path="/tmp/bridge.cts") + assert self._fake is not None + handler_started = threading.Event() + release_handler = threading.Event() + + def _slow_handler(_params: dict) -> dict: + handler_started.set() + release_handler.wait(timeout=2.0) + return {"text": "host:done", "model": "host-test"} + + client.register_host_handler("host.llm.complete", _slow_handler) + self._fake.stdout._enqueue( + { + "jsonrpc": "2.0", + "id": "srv-slow", + "method": "host.llm.complete", + "params": {"messages": [{"role": "user", "content": "slow"}]}, + } + ) + self.assertTrue(handler_started.wait(timeout=0.5)) + + try: + response = client.request( + "turn.start", + { + "sessionId": "hermes:session:1", + "userText": "foreground recall", + }, + timeout=0.5, + ) + self.assertIn("foreground recall", response["injectedContext"]) + finally: + release_handler.set() + + reverse_response = self._wait_for_client_write( + lambda msg: msg.get("id") == "srv-slow" + ) + self.assertEqual(reverse_response["result"]["text"], "host:done") + client.close() + + def test_reverse_handler_queue_rejects_overload_without_blocking_reader(self) -> None: + client = MemosBridgeClient(bridge_path="/tmp/bridge.cts") + assert self._fake is not None + handler_started = threading.Event() + release_handler = threading.Event() + + def _slow_handler(_params: dict) -> dict: + handler_started.set() + release_handler.wait(timeout=2.0) + return {"text": "done"} + + client.register_host_handler("host.llm.complete", _slow_handler) + self._fake.stdout._enqueue( + { + "jsonrpc": "2.0", + "id": "srv-running", + "method": "host.llm.complete", + "params": {}, + } + ) + self.assertTrue(handler_started.wait(timeout=0.5)) + + overflow_id = "srv-overflow" + for index in range(bridge_client_mod.HOST_HANDLER_QUEUE_CAPACITY + 1): + rpc_id = ( + overflow_id + if index == bridge_client_mod.HOST_HANDLER_QUEUE_CAPACITY + else f"srv-{index}" + ) + self._fake.stdout._enqueue( + { + "jsonrpc": "2.0", + "id": rpc_id, + "method": "host.llm.complete", + "params": {}, + } + ) + + try: + response = self._wait_for_client_write( + lambda msg: msg.get("id") == overflow_id + ) + self.assertEqual(response["error"]["data"]["code"], "host_handler_busy") + finally: + client.close() + release_handler.set() + + def test_close_does_not_wait_for_a_running_reverse_handler(self) -> None: + """An uncooperative host callback must not extend bridge shutdown.""" + client = MemosBridgeClient(bridge_path="/tmp/bridge.cts") + assert self._fake is not None + handler_started = threading.Event() + release_handler = threading.Event() + + def _slow_handler(_params: dict) -> dict: + handler_started.set() + release_handler.wait(timeout=2.0) + return {"text": "late", "model": "host-test"} + + client.register_host_handler("host.llm.complete", _slow_handler) + self._fake.stdout._enqueue( + { + "jsonrpc": "2.0", + "id": "srv-close", + "method": "host.llm.complete", + "params": {}, + } + ) + self.assertTrue(handler_started.wait(timeout=0.5)) + + started = time.monotonic() + try: + client.close() + finally: + release_handler.set() + self.assertLess(time.monotonic() - started, 0.5) + def test_reader_exit_marks_pending_as_transport_closed(self) -> None: """R1 (#2028): reader thread EOF must wake pending waiters with transport_closed instead of leaving them parked on their From 2a8e1ce61cd079835fb5e02bc3965efd6f4c4939 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=81=E5=9C=A8=E5=90=B5=E7=9D=80=E5=90=83=E7=B3=96?= Date: Tue, 4 Aug 2026 17:26:42 +0800 Subject: [PATCH 16/18] fix(plugin): honor long Retry-After cooldowns --- .../core/embedding/embedder.ts | 8 + .../core/embedding/fetcher.ts | 128 +++++++++++++-- .../core/embedding/providers/cohere.ts | 4 +- .../core/embedding/providers/gemini.ts | 4 +- .../core/embedding/providers/mistral.ts | 4 +- .../core/embedding/providers/openai.ts | 4 +- .../core/embedding/providers/voyage.ts | 4 +- .../core/embedding/retry-worker.ts | 23 ++- .../core/embedding/types.ts | 9 +- apps/memos-local-plugin/core/llm/client.ts | 11 ++ apps/memos-local-plugin/core/llm/fetcher.ts | 127 +++++++++++++-- .../core/llm/providers/anthropic.ts | 4 +- .../core/llm/providers/bedrock.ts | 4 +- .../core/llm/providers/gemini.ts | 4 +- .../core/llm/providers/openai.ts | 4 +- apps/memos-local-plugin/core/llm/types.ts | 9 +- .../core/retrieval/llm-filter.ts | 2 + .../core/retrieval/retrieve.ts | 7 +- .../core/retrieval/types.ts | 2 +- .../core/util/retry-after.ts | 147 +++++++++++++++++- .../tests/unit/embedding/embedder.test.ts | 52 ++++++- .../tests/unit/embedding/fetcher.test.ts | 55 +++++++ .../tests/unit/embedding/retry-worker.test.ts | 34 ++++ .../tests/unit/llm/client.test.ts | 39 +++++ .../tests/unit/llm/fetcher.test.ts | 140 ++++++++++++++++- .../tests/unit/util/retry-after.test.ts | 66 +++++++- 26 files changed, 834 insertions(+), 61 deletions(-) diff --git a/apps/memos-local-plugin/core/embedding/embedder.ts b/apps/memos-local-plugin/core/embedding/embedder.ts index 0b21209f4..5d035b6ce 100644 --- a/apps/memos-local-plugin/core/embedding/embedder.ts +++ b/apps/memos-local-plugin/core/embedding/embedder.ts @@ -18,6 +18,7 @@ import { ERROR_CODES, MemosError } from "../../agent-contract/errors.js"; import { rootLogger } from "../logger/index.js"; import type { Logger } from "../logger/types.js"; import type { EmbeddingVector } from "../types.js"; +import { extractRetryDiagnostics } from "../util/retry-after.js"; import { LruEmbedCache, NullEmbedCache, @@ -88,6 +89,10 @@ export function createEmbedderWithProvider( code?: string; at?: number; durationMs?: number; + retryAfterMs?: number; + retryAt?: number; + retryDecision?: "wait" | "defer" | "stop"; + retryReason?: string; }): void { if (!config.onStatus) return; try { @@ -185,6 +190,7 @@ export function createEmbedderWithProvider( config, log: providerCtxLog, signal: options?.signal, + deadlineAt: options?.deadlineAt, }; raw = await provider.embed(texts, role, ctx); // Record success but DO NOT clear `lastError` — the viewer @@ -229,6 +235,7 @@ export function createEmbedderWithProvider( message: errMessage, code: err instanceof MemosError ? err.code : undefined, at: errAt, + ...extractRetryDiagnostics(err instanceof MemosError ? err.details : undefined), }); } catch { /* sink errors are non-fatal */ @@ -242,6 +249,7 @@ export function createEmbedderWithProvider( code: err instanceof MemosError ? err.code : undefined, at: errAt, durationMs: errAt - startedAt, + ...extractRetryDiagnostics(err instanceof MemosError ? err.details : undefined), }); throw err instanceof MemosError ? err diff --git a/apps/memos-local-plugin/core/embedding/fetcher.ts b/apps/memos-local-plugin/core/embedding/fetcher.ts index c17b676a6..303dae28e 100644 --- a/apps/memos-local-plugin/core/embedding/fetcher.ts +++ b/apps/memos-local-plugin/core/embedding/fetcher.ts @@ -8,7 +8,15 @@ */ import { ERROR_CODES, MemosError } from "../../agent-contract/errors.js"; -import { parseRetryAfterMs, retryDelayMs, waitForRetry } from "../util/retry-after.js"; +import { + getRetryCooldown, + parseRetryAfterMs, + planRetry, + recordRetryCooldown, + retryCooldownKey, + type RetryPlan, + waitForRetry, +} from "../util/retry-after.js"; import type { EmbeddingProviderName, ProviderLogger } from "./types.js"; export interface HttpPostOpts { @@ -18,6 +26,10 @@ export interface HttpPostOpts { timeoutMs?: number; maxRetries?: number; signal?: AbortSignal; + /** Absolute end-to-end deadline. Unlike timeoutMs, this is not renewed per attempt. */ + deadlineAt?: number; + /** Model/deployment scope; prevents one model cooldown from blocking another. */ + cooldownScope?: string; provider: EmbeddingProviderName; log: ProviderLogger; } @@ -27,11 +39,33 @@ export async function httpPostJson(opts: HttpPostOpts): Promise< const maxRetries = opts.maxRetries ?? 2; let attempt = 0; let lastErr: unknown = null; + const cooldownKey = retryCooldownKey("embedding", opts.provider, opts.url, opts.cooldownScope); while (attempt <= maxRetries) { attempt++; const start = Date.now(); try { + const cooldown = getRetryCooldown(cooldownKey, start); + if (cooldown) { + const details = { + provider: opts.provider, + url: opts.url, + status: cooldown.status, + attempt, + maxRetries, + retryAfterMs: cooldown.retryAfterMs, + retryAt: cooldown.retryAt, + retryDecision: "defer", + retryReason: "cooldown_active", + remainingDeadlineMs: remainingDeadlineMs(opts.deadlineAt, start), + }; + opts.log.warn("http.retry_cooldown", details); + throw new MemosError( + ERROR_CODES.EMBEDDING_UNAVAILABLE, + `${opts.provider} is cooling down until ${new Date(cooldown.retryAt).toISOString()}`, + details, + ); + } const signal = mergeSignals(opts.signal, AbortSignal.timeout(timeoutMs)); const resp = await fetch(opts.url, { method: "POST", @@ -50,6 +84,13 @@ export async function httpPostJson(opts: HttpPostOpts): Promise< const retryAfterMs = resp.status === 429 || resp.status === 503 ? parseRetryAfterMs(resp.headers.get("Retry-After")) : null; + if (retryAfterMs !== null) { + recordRetryCooldown(cooldownKey, { + retryAfterMs, + retryAt: Date.now() + retryAfterMs, + status: resp.status, + }); + } opts.log.warn("http.non_ok", { url: opts.url, status: resp.status, @@ -59,13 +100,43 @@ export async function httpPostJson(opts: HttpPostOpts): Promise< durationMs: Date.now() - start, }); if (transient && attempt <= maxRetries) { - await backoff(attempt, retryAfterMs, opts.signal); + const plan = planRetry({ + attempt, + baseMs: 200, + jitterMaxMs: 100, + retryAfterMs, + deadlineAt: opts.deadlineAt, + }); + const retryDetails = retryPlanDetails(plan, opts, maxRetries, resp.status, attempt); + if (plan.action === "defer") { + opts.log.warn("http.retry_deferred", retryDetails); + throw new MemosError( + ERROR_CODES.EMBEDDING_UNAVAILABLE, + `HTTP ${resp.status} from ${opts.provider}; retry deferred until ${new Date(plan.retryAt).toISOString()}`, + retryDetails, + ); + } + opts.log.warn("http.retry_scheduled", retryDetails); + await waitForRetry(plan.delayMs, opts.signal); continue; } throw new MemosError( ERROR_CODES.EMBEDDING_UNAVAILABLE, `HTTP ${resp.status} from ${opts.provider}`, - { provider: opts.provider, url: opts.url, status: resp.status, body: text }, + { + provider: opts.provider, + url: opts.url, + status: resp.status, + body: text, + ...(retryAfterMs === null + ? {} + : { + retryAfterMs, + retryAt: Date.now() + retryAfterMs, + retryDecision: "stop", + retryReason: "retries_exhausted", + }), + }, ); } @@ -88,7 +159,23 @@ export async function httpPostJson(opts: HttpPostOpts): Promise< durationMs: Date.now() - start, }); if (transient && attempt <= maxRetries) { - await backoff(attempt, null, opts.signal); + const plan = planRetry({ + attempt, + baseMs: 200, + jitterMaxMs: 100, + deadlineAt: opts.deadlineAt, + }); + const retryDetails = retryPlanDetails(plan, opts, maxRetries, null, attempt); + if (plan.action === "defer") { + opts.log.warn("http.retry_deferred", retryDetails); + throw new MemosError( + ERROR_CODES.EMBEDDING_UNAVAILABLE, + `${opts.provider} retry cannot fit the request deadline`, + retryDetails, + ); + } + opts.log.warn("http.retry_scheduled", retryDetails); + await waitForRetry(plan.delayMs, opts.signal); continue; } throw new MemosError( @@ -128,15 +215,32 @@ function isTransientError(err: unknown): boolean { return false; } -async function backoff( +function retryPlanDetails( + plan: RetryPlan, + opts: HttpPostOpts, + maxRetries: number, + status: number | null, attempt: number, - retryAfterMs: number | null, - signal?: AbortSignal, -): Promise { - await waitForRetry( - retryDelayMs({ attempt, baseMs: 200, jitterMaxMs: 100, retryAfterMs }), - signal, - ); +): Record { + return { + provider: opts.provider, + url: opts.url, + status, + attempt, + maxRetries, + backoffMs: plan.backoffMs, + plannedDelayMs: plan.delayMs, + retryAfterMs: plan.retryAfterMs, + retryAt: plan.retryAt, + retrySource: plan.source, + retryDecision: plan.action, + ...(plan.action === "defer" ? { retryReason: plan.reason } : {}), + remainingDeadlineMs: remainingDeadlineMs(opts.deadlineAt), + }; +} + +function remainingDeadlineMs(deadlineAt?: number, nowMs: number = Date.now()): number | null { + return deadlineAt === undefined ? null : Math.max(0, deadlineAt - nowMs); } function mergeSignals(a: AbortSignal | undefined, b: AbortSignal): AbortSignal { diff --git a/apps/memos-local-plugin/core/embedding/providers/cohere.ts b/apps/memos-local-plugin/core/embedding/providers/cohere.ts index 891cd4506..58214d341 100644 --- a/apps/memos-local-plugin/core/embedding/providers/cohere.ts +++ b/apps/memos-local-plugin/core/embedding/providers/cohere.ts @@ -21,7 +21,7 @@ export class CohereEmbeddingProvider implements EmbeddingProvider { readonly name: EmbeddingProviderName = "cohere"; async embed(texts: string[], role: EmbedRole, ctx: ProviderCallCtx): Promise { - const { config, log, signal } = ctx; + const { config, log, signal, deadlineAt } = ctx; if (!config.apiKey) { throw new MemosError( ERROR_CODES.EMBEDDING_UNAVAILABLE, @@ -49,6 +49,8 @@ export class CohereEmbeddingProvider implements EmbeddingProvider { timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, signal, + deadlineAt, + cooldownScope: config.model, provider: this.name, log, }); diff --git a/apps/memos-local-plugin/core/embedding/providers/gemini.ts b/apps/memos-local-plugin/core/embedding/providers/gemini.ts index 91d97acba..68ba22708 100644 --- a/apps/memos-local-plugin/core/embedding/providers/gemini.ts +++ b/apps/memos-local-plugin/core/embedding/providers/gemini.ts @@ -22,7 +22,7 @@ export class GeminiEmbeddingProvider implements EmbeddingProvider { readonly name: EmbeddingProviderName = "gemini"; async embed(texts: string[], role: EmbedRole, ctx: ProviderCallCtx): Promise { - const { config, log, signal } = ctx; + const { config, log, signal, deadlineAt } = ctx; if (!config.apiKey) { throw new MemosError( ERROR_CODES.EMBEDDING_UNAVAILABLE, @@ -50,6 +50,8 @@ export class GeminiEmbeddingProvider implements EmbeddingProvider { timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, signal, + deadlineAt, + cooldownScope: config.model, provider: this.name, log, }); diff --git a/apps/memos-local-plugin/core/embedding/providers/mistral.ts b/apps/memos-local-plugin/core/embedding/providers/mistral.ts index 21f50769f..7eace8ab7 100644 --- a/apps/memos-local-plugin/core/embedding/providers/mistral.ts +++ b/apps/memos-local-plugin/core/embedding/providers/mistral.ts @@ -23,7 +23,7 @@ export class MistralEmbeddingProvider implements EmbeddingProvider { readonly name: EmbeddingProviderName = "mistral"; async embed(texts: string[], _role: EmbedRole, ctx: ProviderCallCtx): Promise { - const { config, log, signal } = ctx; + const { config, log, signal, deadlineAt } = ctx; if (!config.apiKey) { throw new MemosError( ERROR_CODES.EMBEDDING_UNAVAILABLE, @@ -46,6 +46,8 @@ export class MistralEmbeddingProvider implements EmbeddingProvider { timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, signal, + deadlineAt, + cooldownScope: config.model, provider: this.name, log, }); diff --git a/apps/memos-local-plugin/core/embedding/providers/openai.ts b/apps/memos-local-plugin/core/embedding/providers/openai.ts index c0df47831..57d852639 100644 --- a/apps/memos-local-plugin/core/embedding/providers/openai.ts +++ b/apps/memos-local-plugin/core/embedding/providers/openai.ts @@ -27,7 +27,7 @@ export class OpenAiEmbeddingProvider implements EmbeddingProvider { readonly name: EmbeddingProviderName = "openai_compatible"; async embed(texts: string[], _role: EmbedRole, ctx: ProviderCallCtx): Promise { - const { config, log, signal } = ctx; + const { config, log, signal, deadlineAt } = ctx; if (!config.apiKey) { throw new MemosError( ERROR_CODES.EMBEDDING_UNAVAILABLE, @@ -53,6 +53,8 @@ export class OpenAiEmbeddingProvider implements EmbeddingProvider { timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, signal, + deadlineAt, + cooldownScope: config.model, provider: this.name, log, }); diff --git a/apps/memos-local-plugin/core/embedding/providers/voyage.ts b/apps/memos-local-plugin/core/embedding/providers/voyage.ts index f89eca832..6f36fe6d8 100644 --- a/apps/memos-local-plugin/core/embedding/providers/voyage.ts +++ b/apps/memos-local-plugin/core/embedding/providers/voyage.ts @@ -23,7 +23,7 @@ export class VoyageEmbeddingProvider implements EmbeddingProvider { readonly name: EmbeddingProviderName = "voyage"; async embed(texts: string[], role: EmbedRole, ctx: ProviderCallCtx): Promise { - const { config, log, signal } = ctx; + const { config, log, signal, deadlineAt } = ctx; if (!config.apiKey) { throw new MemosError( ERROR_CODES.EMBEDDING_UNAVAILABLE, @@ -50,6 +50,8 @@ export class VoyageEmbeddingProvider implements EmbeddingProvider { timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, signal, + deadlineAt, + cooldownScope: config.model, provider: this.name, log, }); diff --git a/apps/memos-local-plugin/core/embedding/retry-worker.ts b/apps/memos-local-plugin/core/embedding/retry-worker.ts index 3c33c2870..34db37bff 100644 --- a/apps/memos-local-plugin/core/embedding/retry-worker.ts +++ b/apps/memos-local-plugin/core/embedding/retry-worker.ts @@ -104,6 +104,11 @@ export function createEmbeddingRetryWorker( const message = err instanceof Error ? err.message : String(err); const at = now(); const terminal = attemptNo >= job.maxAttempts; + const providerRetryAt = retryAtFromError(err, at); + const nextAttemptAt = Math.max( + at + backoffMs(attemptNo), + providerRetryAt ?? 0, + ); const recorded = terminal ? deps.repos.embeddingRetryQueue.markFailedClaimed(job.id, { ...claim, @@ -114,7 +119,7 @@ export function createEmbeddingRetryWorker( : deps.repos.embeddingRetryQueue.markRetryClaimed(job.id, { ...claim, attempts: attemptNo, - nextAttemptAt: at + backoffMs(attemptNo), + nextAttemptAt, error: message, now: at, }); @@ -122,7 +127,10 @@ export function createEmbeddingRetryWorker( deps.log.debug("embedding_retry.stale_failure_ignored", { jobId: job.id, terminal }); return; } - emitFailure(job, attemptNo, message, terminal, at); + emitFailure(job, attemptNo, message, terminal, at, { + providerRetryAt, + nextAttemptAt: terminal ? null : nextAttemptAt, + }); } } @@ -154,6 +162,7 @@ export function createEmbeddingRetryWorker( message: string, terminal: boolean, at: number, + retry: { providerRetryAt: number | null; nextAttemptAt: number | null }, ): void { const payload = { kind: "embedding.retry_failed", @@ -165,6 +174,8 @@ export function createEmbeddingRetryWorker( maxAttempts: job.maxAttempts, terminal, message, + providerRetryAt: retry.providerRetryAt, + nextAttemptAt: retry.nextAttemptAt, }; deps.log.warn("embedding_retry.failed", payload); try { @@ -207,6 +218,14 @@ export function createEmbeddingRetryWorker( }; } +function retryAtFromError(err: unknown, nowMs: number): number | null { + if (!err || typeof err !== "object") return null; + const details = (err as { details?: unknown }).details; + if (!details || typeof details !== "object") return null; + const retryAt = Number((details as { retryAt?: unknown }).retryAt); + return Number.isSafeInteger(retryAt) && retryAt > nowMs ? retryAt : null; +} + function backoffMs(attemptNo: number): number { return Math.min(MAX_BACKOFF_MS, BASE_BACKOFF_MS * 2 ** Math.max(0, attemptNo - 1)); } diff --git a/apps/memos-local-plugin/core/embedding/types.ts b/apps/memos-local-plugin/core/embedding/types.ts index 0c548cc92..95726703c 100644 --- a/apps/memos-local-plugin/core/embedding/types.ts +++ b/apps/memos-local-plugin/core/embedding/types.ts @@ -6,6 +6,7 @@ */ import type { EmbeddingVector } from "../types.js"; +import type { RetryDiagnosticDetails } from "../util/retry-after.js"; // ─── Config ────────────────────────────────────────────────────────────────── @@ -62,7 +63,7 @@ export interface EmbeddingConfig { onStatus?: (detail: EmbeddingStatusDetail) => void; } -export interface EmbeddingErrorDetail { +export interface EmbeddingErrorDetail extends RetryDiagnosticDetails { kind: "embedding"; provider: EmbeddingProviderName | string; model: string; @@ -73,7 +74,7 @@ export interface EmbeddingErrorDetail { at?: number; } -export interface EmbeddingStatusDetail { +export interface EmbeddingStatusDetail extends RetryDiagnosticDetails { kind: "embedding"; status: "ok" | "error"; provider: EmbeddingProviderName | string; @@ -128,6 +129,8 @@ export interface ProviderCallCtx { log: ProviderLogger; /** AbortSignal honored across HTTP + native calls. */ signal?: AbortSignal; + /** Absolute end-to-end deadline shared across provider retry attempts. */ + deadlineAt?: number; } export interface ProviderLogger { @@ -186,6 +189,8 @@ export interface Embedder { export interface EmbedCallOptions { signal?: AbortSignal; + /** Absolute end-to-end deadline shared across provider retry attempts. */ + deadlineAt?: number; } // ─── Errors ────────────────────────────────────────────────────────────────── diff --git a/apps/memos-local-plugin/core/llm/client.ts b/apps/memos-local-plugin/core/llm/client.ts index 6bedafa70..ee456ac12 100644 --- a/apps/memos-local-plugin/core/llm/client.ts +++ b/apps/memos-local-plugin/core/llm/client.ts @@ -18,6 +18,7 @@ import { ERROR_CODES, MemosError } from "../../agent-contract/errors.js"; import { rootLogger } from "../logger/index.js"; import type { Logger } from "../logger/types.js"; +import { extractRetryDiagnostics } from "../util/retry-after.js"; import { getHostLlmBridge } from "./host-bridge.js"; import { buildJsonSystemHint, parseLlmJson } from "./json-mode.js"; import { AnthropicLlmProvider } from "./providers/anthropic.js"; @@ -292,6 +293,7 @@ export function createLlmClientWithProvider( }, log: pLog, signal: opts?.signal, + deadlineAt: opts?.deadlineAt, }; } @@ -370,6 +372,7 @@ export function createLlmClientWithProvider( model: config.model, message: summarizeErrMessage(hostErr), code: hostErr instanceof MemosError ? hostErr.code : undefined, + ...extractRetryDiagnostics(hostErr instanceof MemosError ? hostErr.details : undefined), at: failAt, durationMs: Date.now() - startedAt, fallbackProvider: "host", @@ -395,6 +398,7 @@ export function createLlmClientWithProvider( model: config.model, message: summarizeErrMessage(err), code: err instanceof MemosError ? err.code : undefined, + ...extractRetryDiagnostics(err instanceof MemosError ? err.details : undefined), at: failAt, durationMs: Date.now() - startedAt, op, @@ -425,6 +429,7 @@ export function createLlmClientWithProvider( message: summarizeErrMessage(err), code: err instanceof MemosError ? err.code : undefined, at: Date.now(), + ...extractRetryDiagnostics(err instanceof MemosError ? err.details : undefined), }); } catch { /* sink errors are non-fatal */ @@ -444,6 +449,10 @@ export function createLlmClientWithProvider( op?: string; episodeId?: string; phase?: string; + retryAfterMs?: number; + retryAt?: number; + retryDecision?: "wait" | "defer" | "stop"; + retryReason?: string; }): void { if (!config.onStatus) return; try { @@ -616,6 +625,7 @@ export function createLlmClientWithProvider( model: config.model, message: summarizeErrMessage(err), code: err instanceof MemosError ? err.code : undefined, + ...extractRetryDiagnostics(err instanceof MemosError ? err.details : undefined), at: failAt, durationMs: Date.now() - start, op: opts?.op ?? "stream", @@ -669,6 +679,7 @@ export function createLlmClientWithProvider( model: config.model, message: summarizeErrMessage(primaryErr), code: primaryErr instanceof MemosError ? primaryErr.code : undefined, + ...extractRetryDiagnostics(primaryErr instanceof MemosError ? primaryErr.details : undefined), at: fallbackAt, durationMs: completion.durationMs, fallbackProvider: "host", diff --git a/apps/memos-local-plugin/core/llm/fetcher.ts b/apps/memos-local-plugin/core/llm/fetcher.ts index ef0d56357..358c73275 100644 --- a/apps/memos-local-plugin/core/llm/fetcher.ts +++ b/apps/memos-local-plugin/core/llm/fetcher.ts @@ -11,7 +11,15 @@ */ import { ERROR_CODES, MemosError } from "../../agent-contract/errors.js"; -import { parseRetryAfterMs, retryDelayMs, waitForRetry } from "../util/retry-after.js"; +import { + getRetryCooldown, + parseRetryAfterMs, + planRetry, + recordRetryCooldown, + retryCooldownKey, + type RetryPlan, + waitForRetry, +} from "../util/retry-after.js"; import type { LlmProviderLogger, LlmProviderName } from "./types.js"; export interface HttpPostOpts { @@ -21,6 +29,10 @@ export interface HttpPostOpts { timeoutMs: number; maxRetries: number; signal?: AbortSignal; + /** Absolute end-to-end deadline. Unlike timeoutMs, this is not renewed per attempt. */ + deadlineAt?: number; + /** Model/deployment scope; prevents one model cooldown from blocking another. */ + cooldownScope?: string; provider: LlmProviderName; log: LlmProviderLogger; onRetry?: (attempt: number) => void; @@ -36,11 +48,33 @@ export async function httpPostJson(opts: HttpPostOpts): Promise< }> { let attempt = 0; let lastErr: unknown = null; + const cooldownKey = retryCooldownKey("llm", opts.provider, opts.url, opts.cooldownScope); while (attempt <= opts.maxRetries) { attempt++; const start = Date.now(); try { + const cooldown = getRetryCooldown(cooldownKey, start); + if (cooldown) { + const details = { + provider: opts.provider, + url: opts.url, + status: cooldown.status, + attempt, + maxRetries: opts.maxRetries, + retryAfterMs: cooldown.retryAfterMs, + retryAt: cooldown.retryAt, + retryDecision: "defer", + retryReason: "cooldown_active", + remainingDeadlineMs: remainingDeadlineMs(opts.deadlineAt, start), + }; + opts.log.warn("http.retry_cooldown", details); + throw new MemosError( + errCodeForStatus(cooldown.status), + `${opts.provider} is cooling down until ${new Date(cooldown.retryAt).toISOString()}`, + details, + ); + } const signal = mergeSignals(opts.signal, AbortSignal.timeout(opts.timeoutMs)); const resp = await fetch(opts.url, { method: "POST", @@ -60,6 +94,13 @@ export async function httpPostJson(opts: HttpPostOpts): Promise< const retryAfterMs = resp.status === 429 || resp.status === 503 ? parseRetryAfterMs(resp.headers.get("Retry-After")) : null; + if (retryAfterMs !== null) { + recordRetryCooldown(cooldownKey, { + retryAfterMs, + retryAt: Date.now() + retryAfterMs, + status: resp.status, + }); + } opts.log.warn("http.non_ok", { status: resp.status, attempt, @@ -69,14 +110,44 @@ export async function httpPostJson(opts: HttpPostOpts): Promise< body: truncateLogBody(text), }); if (transient && attempt <= opts.maxRetries) { + const plan = planRetry({ + attempt, + baseMs: 250, + jitterMaxMs: 120, + retryAfterMs, + deadlineAt: opts.deadlineAt, + }); + const retryDetails = retryPlanDetails(plan, opts, resp.status, attempt); + if (plan.action === "defer") { + opts.log.warn("http.retry_deferred", retryDetails); + throw new MemosError( + errCodeForStatus(resp.status), + `HTTP ${resp.status} from ${opts.provider}; retry deferred until ${new Date(plan.retryAt).toISOString()}`, + retryDetails, + ); + } + opts.log.warn("http.retry_scheduled", retryDetails); opts.onRetry?.(attempt); - await backoff(attempt, retryAfterMs, opts.signal); + await waitForRetry(plan.delayMs, opts.signal); continue; } throw new MemosError( errCodeForStatus(resp.status), `HTTP ${resp.status} from ${opts.provider}`, - { provider: opts.provider, url: opts.url, status: resp.status, body: text }, + { + provider: opts.provider, + url: opts.url, + status: resp.status, + body: text, + ...(retryAfterMs === null + ? {} + : { + retryAfterMs, + retryAt: Date.now() + retryAfterMs, + retryDecision: "stop", + retryReason: "retries_exhausted", + }), + }, ); } @@ -106,8 +177,24 @@ export async function httpPostJson(opts: HttpPostOpts): Promise< err: toErrDetail(err), }); if ((transient || timedOut) && attempt <= opts.maxRetries) { + const plan = planRetry({ + attempt, + baseMs: 250, + jitterMaxMs: 120, + deadlineAt: opts.deadlineAt, + }); + const retryDetails = retryPlanDetails(plan, opts, null, attempt); + if (plan.action === "defer") { + opts.log.warn("http.retry_deferred", retryDetails); + throw new MemosError( + timedOut ? ERROR_CODES.LLM_TIMEOUT : ERROR_CODES.LLM_UNAVAILABLE, + `${opts.provider} retry cannot fit the request deadline`, + retryDetails, + ); + } + opts.log.warn("http.retry_scheduled", retryDetails); opts.onRetry?.(attempt); - await backoff(attempt, null, opts.signal); + await waitForRetry(plan.delayMs, opts.signal); continue; } if (timedOut) { @@ -257,15 +344,31 @@ function isTimeout(err: unknown): boolean { return false; } -async function backoff( +function retryPlanDetails( + plan: RetryPlan, + opts: HttpPostOpts, + status: number | null, attempt: number, - retryAfterMs: number | null, - signal?: AbortSignal, -): Promise { - await waitForRetry( - retryDelayMs({ attempt, baseMs: 250, jitterMaxMs: 120, retryAfterMs }), - signal, - ); +): Record { + return { + provider: opts.provider, + url: opts.url, + status, + attempt, + maxRetries: opts.maxRetries, + backoffMs: plan.backoffMs, + plannedDelayMs: plan.delayMs, + retryAfterMs: plan.retryAfterMs, + retryAt: plan.retryAt, + retrySource: plan.source, + retryDecision: plan.action, + ...(plan.action === "defer" ? { retryReason: plan.reason } : {}), + remainingDeadlineMs: remainingDeadlineMs(opts.deadlineAt), + }; +} + +function remainingDeadlineMs(deadlineAt?: number, nowMs: number = Date.now()): number | null { + return deadlineAt === undefined ? null : Math.max(0, deadlineAt - nowMs); } function mergeSignals(a: AbortSignal | undefined, b: AbortSignal): AbortSignal { diff --git a/apps/memos-local-plugin/core/llm/providers/anthropic.ts b/apps/memos-local-plugin/core/llm/providers/anthropic.ts index 66a9ee446..6c510ef75 100644 --- a/apps/memos-local-plugin/core/llm/providers/anthropic.ts +++ b/apps/memos-local-plugin/core/llm/providers/anthropic.ts @@ -31,7 +31,7 @@ export class AnthropicLlmProvider implements LlmProvider { opts: ProviderCallInput, ctx: LlmProviderCtx, ): Promise { - const { config, log, signal } = ctx; + const { config, log, signal, deadlineAt } = ctx; if (!config.apiKey) { throw new MemosError( ERROR_CODES.LLM_UNAVAILABLE, @@ -68,6 +68,8 @@ export class AnthropicLlmProvider implements LlmProvider { timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, signal, + deadlineAt, + cooldownScope: config.model, provider: this.name, log, }); diff --git a/apps/memos-local-plugin/core/llm/providers/bedrock.ts b/apps/memos-local-plugin/core/llm/providers/bedrock.ts index 7c00470e7..d956b7111 100644 --- a/apps/memos-local-plugin/core/llm/providers/bedrock.ts +++ b/apps/memos-local-plugin/core/llm/providers/bedrock.ts @@ -36,7 +36,7 @@ export class BedrockLlmProvider implements LlmProvider { opts: ProviderCallInput, ctx: LlmProviderCtx, ): Promise { - const { config, log, signal } = ctx; + const { config, log, signal, deadlineAt } = ctx; if (!config.endpoint || config.endpoint.length === 0) { throw new MemosError( ERROR_CODES.LLM_UNAVAILABLE, @@ -85,6 +85,8 @@ export class BedrockLlmProvider implements LlmProvider { timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, signal, + deadlineAt, + cooldownScope: config.model, provider: this.name, log, }); diff --git a/apps/memos-local-plugin/core/llm/providers/gemini.ts b/apps/memos-local-plugin/core/llm/providers/gemini.ts index 4e6273b55..6bfa323eb 100644 --- a/apps/memos-local-plugin/core/llm/providers/gemini.ts +++ b/apps/memos-local-plugin/core/llm/providers/gemini.ts @@ -39,7 +39,7 @@ export class GeminiLlmProvider implements LlmProvider { opts: ProviderCallInput, ctx: LlmProviderCtx, ): Promise { - const { config, log, signal } = ctx; + const { config, log, signal, deadlineAt } = ctx; if (!config.apiKey) { throw new MemosError( ERROR_CODES.LLM_UNAVAILABLE, @@ -59,6 +59,8 @@ export class GeminiLlmProvider implements LlmProvider { timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, signal, + deadlineAt, + cooldownScope: config.model, provider: this.name, log, }); diff --git a/apps/memos-local-plugin/core/llm/providers/openai.ts b/apps/memos-local-plugin/core/llm/providers/openai.ts index d8a5a20af..562521756 100644 --- a/apps/memos-local-plugin/core/llm/providers/openai.ts +++ b/apps/memos-local-plugin/core/llm/providers/openai.ts @@ -51,7 +51,7 @@ export class OpenAiLlmProvider implements LlmProvider { opts: ProviderCallInput, ctx: LlmProviderCtx, ): Promise { - const { config, log, signal } = ctx; + const { config, log, signal, deadlineAt } = ctx; const url = normalizeEndpoint( config.endpoint && config.endpoint.length > 0 ? config.endpoint @@ -93,6 +93,8 @@ export class OpenAiLlmProvider implements LlmProvider { timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, signal, + deadlineAt, + cooldownScope: config.model, provider: this.name, log, }); diff --git a/apps/memos-local-plugin/core/llm/types.ts b/apps/memos-local-plugin/core/llm/types.ts index a37a2677d..2dd761f32 100644 --- a/apps/memos-local-plugin/core/llm/types.ts +++ b/apps/memos-local-plugin/core/llm/types.ts @@ -6,6 +6,7 @@ */ import type { ReasoningConfig as ConfigReasoningConfig } from "../config/schema.js"; +import type { RetryDiagnosticDetails } from "../util/retry-after.js"; // ─── Providers & config ────────────────────────────────────────────────────── @@ -88,7 +89,7 @@ export interface LlmCircuitBreakerConfig { now?: () => number; } -export interface LlmErrorDetail { +export interface LlmErrorDetail extends RetryDiagnosticDetails { provider: LlmProviderName | string; model: string; message: string; @@ -105,7 +106,7 @@ export interface LlmErrorDetail { role?: "llm" | "skillEvolver"; } -export interface LlmStatusDetail { +export interface LlmStatusDetail extends RetryDiagnosticDetails { status: "ok" | "fallback" | "error" | "circuit_open"; provider: LlmProviderName | string; model: string; @@ -149,6 +150,8 @@ export interface LlmCallOptions { maxTokens?: number; /** Per-call timeout. */ timeoutMs?: number; + /** Absolute end-to-end deadline shared across provider retry attempts. */ + deadlineAt?: number; /** AbortSignal honored across HTTP + host-bridge calls. */ signal?: AbortSignal; /** @@ -213,6 +216,8 @@ export interface LlmProviderCtx { log: LlmProviderLogger; /** Call abort signal; providers must honor it. */ signal?: AbortSignal; + /** Absolute end-to-end deadline; providers must not renew it per retry. */ + deadlineAt?: number; } export interface LlmProviderLogger { diff --git a/apps/memos-local-plugin/core/retrieval/llm-filter.ts b/apps/memos-local-plugin/core/retrieval/llm-filter.ts index 87dc2520e..f665cb954 100644 --- a/apps/memos-local-plugin/core/retrieval/llm-filter.ts +++ b/apps/memos-local-plugin/core/retrieval/llm-filter.ts @@ -50,6 +50,7 @@ export interface FilterDeps { llm: LlmClient | null; log: Logger; timeoutMs?: number; + deadlineAt?: number; signal?: AbortSignal; config: Pick< RetrievalConfig, @@ -153,6 +154,7 @@ ${list}`, episodeId: input.episodeId, temperature: 0, timeoutMs: deps.timeoutMs, + deadlineAt: deps.deadlineAt, signal: deps.signal, // Output is only ordered indices + one bool, but the list can // legitimately be as long as the ranked candidates. diff --git a/apps/memos-local-plugin/core/retrieval/retrieve.ts b/apps/memos-local-plugin/core/retrieval/retrieve.ts index 22a7df107..ff95debf4 100644 --- a/apps/memos-local-plugin/core/retrieval/retrieve.ts +++ b/apps/memos-local-plugin/core/retrieval/retrieve.ts @@ -263,7 +263,10 @@ async function runAll( }; const queryVec = compiled.text ? await deps.embedder - .embed(compiled.text, "query", { signal: opts.signal }) + .embed(compiled.text, "query", { + signal: opts.signal, + deadlineAt: opts.deadlineAt, + }) .then((vec) => { embeddingStats.ok = true; return vec; @@ -423,6 +426,7 @@ async function runAll( log, config: deps.config, signal: opts.signal, + deadlineAt: opts.deadlineAt, timeoutMs: filterTimeoutMs(opts.deadlineAt), }, ); @@ -482,6 +486,7 @@ async function runAll( log, config: deps.config, signal: opts.signal, + deadlineAt: opts.deadlineAt, timeoutMs: filterTimeoutMs(opts.deadlineAt), }, ); diff --git a/apps/memos-local-plugin/core/retrieval/types.ts b/apps/memos-local-plugin/core/retrieval/types.ts index bcc9bb7bb..fe700c611 100644 --- a/apps/memos-local-plugin/core/retrieval/types.ts +++ b/apps/memos-local-plugin/core/retrieval/types.ts @@ -682,7 +682,7 @@ export interface RetrievalEmbedder { embed: ( text: string, role?: "query" | "document", - options?: { signal?: AbortSignal }, + options?: { signal?: AbortSignal; deadlineAt?: number }, ) => Promise; } diff --git a/apps/memos-local-plugin/core/util/retry-after.ts b/apps/memos-local-plugin/core/util/retry-after.ts index 0ca668514..e580e5f7b 100644 --- a/apps/memos-local-plugin/core/util/retry-after.ts +++ b/apps/memos-local-plugin/core/util/retry-after.ts @@ -1,5 +1,38 @@ /** Parse RFC 9110 Retry-After delay-seconds or HTTP-date into milliseconds. */ -export const MAX_RETRY_DELAY_MS = 30_000; +export const MAX_INLINE_RETRY_DELAY_MS = 30_000; +/** @deprecated Use MAX_INLINE_RETRY_DELAY_MS. */ +export const MAX_RETRY_DELAY_MS = MAX_INLINE_RETRY_DELAY_MS; + +export type RetryDeferReason = + | "deadline_insufficient" + | "retry_after_too_long"; + +export interface RetryPlanBase { + backoffMs: number; + delayMs: number; + retryAfterMs: number | null; + retryAt: number; + source: "backoff" | "retry_after"; +} + +export type RetryPlan = + | (RetryPlanBase & { action: "wait" }) + | (RetryPlanBase & { action: "defer"; reason: RetryDeferReason }); + +export interface RetryCooldown { + retryAfterMs: number; + retryAt: number; + status: number; +} + +export interface RetryDiagnosticDetails { + retryAfterMs?: number; + retryAt?: number; + retryDecision?: "wait" | "defer" | "stop"; + retryReason?: string; +} + +const retryCooldowns = new Map(); export function parseRetryAfterMs( value: string | null | undefined, @@ -30,11 +63,117 @@ export function retryDelayMs(input: { maxDelayMs?: number; random?: () => number; }): number { + const plan = planRetry({ + ...input, + maxInlineDelayMs: input.maxDelayMs, + }); + return plan.delayMs; +} + +/** + * Decide whether a retry can happen inline without violating Retry-After. + * + * Provider Retry-After values are never clamped downward. When the earliest + * legal retry cannot fit the inline wait or request deadline, callers must + * defer/fallback and carry retryAt into their recovery path. + */ +export function planRetry(input: { + attempt: number; + baseMs: number; + jitterMaxMs: number; + retryAfterMs?: number | null; + maxInlineDelayMs?: number; + deadlineAt?: number; + nowMs?: number; + random?: () => number; +}): RetryPlan { + const nowMs = input.nowMs ?? Date.now(); const random = input.random ?? Math.random; const jitter = Math.floor(random() * input.jitterMaxMs); - const exponential = input.baseMs * 2 ** Math.max(0, input.attempt - 1) + jitter; - const requested = Math.max(exponential, input.retryAfterMs ?? 0); - return Math.min(requested, input.maxDelayMs ?? MAX_RETRY_DELAY_MS); + const rawBackoff = input.baseMs * 2 ** Math.max(0, input.attempt - 1) + jitter; + const maxInlineDelayMs = input.maxInlineDelayMs ?? MAX_INLINE_RETRY_DELAY_MS; + const backoffMs = Math.min(rawBackoff, maxInlineDelayMs); + const retryAfterMs = input.retryAfterMs ?? null; + const delayMs = Math.max(backoffMs, retryAfterMs ?? 0); + const retryAt = nowMs + delayMs; + const source = retryAfterMs !== null && retryAfterMs >= backoffMs + ? "retry_after" as const + : "backoff" as const; + const base: RetryPlanBase = { + backoffMs, + delayMs, + retryAfterMs, + retryAt, + source, + }; + + if (retryAfterMs !== null && retryAfterMs > maxInlineDelayMs) { + return { ...base, action: "defer", reason: "retry_after_too_long" }; + } + if (input.deadlineAt !== undefined && retryAt > input.deadlineAt) { + return { ...base, action: "defer", reason: "deadline_insufficient" }; + } + return { ...base, action: "wait" }; +} + +export function retryCooldownKey( + kind: "llm" | "embedding", + provider: string, + url: string, + scope: string = "", +): string { + return `${kind}\u0000${provider}\u0000${url}\u0000${scope}`; +} + +/** Extend a provider cooldown monotonically; a shorter later response cannot weaken it. */ +export function recordRetryCooldown(key: string, cooldown: RetryCooldown): void { + const current = retryCooldowns.get(key); + if (!current || cooldown.retryAt > current.retryAt) { + retryCooldowns.set(key, { ...cooldown }); + } +} + +export function getRetryCooldown( + key: string, + nowMs: number = Date.now(), +): RetryCooldown | null { + const cooldown = retryCooldowns.get(key); + if (!cooldown) return null; + if (cooldown.retryAt <= nowMs) { + retryCooldowns.delete(key); + return null; + } + return { ...cooldown }; +} + +/** Test/runtime-reset hook; plugin shutdown does not need to await cooldown state. */ +export function clearRetryCooldowns(): void { + retryCooldowns.clear(); +} + +/** Copy only bounded, machine-readable retry fields from an error detail bag. */ +export function extractRetryDiagnostics( + details: Record | undefined, +): RetryDiagnosticDetails { + if (!details) return {}; + const diagnostic: RetryDiagnosticDetails = {}; + if (typeof details.retryAfterMs === "number" && Number.isFinite(details.retryAfterMs)) { + diagnostic.retryAfterMs = details.retryAfterMs; + } + if (typeof details.retryAt === "number" && Number.isFinite(details.retryAt)) { + diagnostic.retryAt = details.retryAt; + } + if ( + details.retryDecision === "wait" + || details.retryDecision === "defer" + || details.retryDecision === "stop" + ) { + diagnostic.retryDecision = details.retryDecision; + } + if (typeof details.retryReason === "string") { + diagnostic.retryReason = details.retryReason; + } + return diagnostic; } /** Abortable retry wait so request cancellation and shutdown do not leave sleepers behind. */ diff --git a/apps/memos-local-plugin/tests/unit/embedding/embedder.test.ts b/apps/memos-local-plugin/tests/unit/embedding/embedder.test.ts index 7c0b6bfff..617bc4a72 100644 --- a/apps/memos-local-plugin/tests/unit/embedding/embedder.test.ts +++ b/apps/memos-local-plugin/tests/unit/embedding/embedder.test.ts @@ -6,8 +6,10 @@ import { initTestLogger } from "../../../core/logger/index.js"; import type { EmbedRole, EmbeddingConfig, + EmbeddingErrorDetail, EmbeddingProvider, EmbeddingProviderName, + EmbeddingStatusDetail, ProviderCallCtx, } from "../../../core/embedding/types.js"; @@ -75,21 +77,22 @@ describe("embedder facade", () => { expect(Array.from(v)).toEqual([3, 97, 0]); // a=97 }); - it("forwards the caller abort signal to the provider", async () => { - const seen: Array = []; + it("forwards the caller abort signal and deadline to the provider", async () => { + const seen: Array> = []; const p: EmbeddingProvider = { name: "openai_compatible", async embed(texts, _role, ctx) { - seen.push(ctx.signal); + seen.push({ signal: ctx.signal, deadlineAt: ctx.deadlineAt }); return texts.map(() => [1, 2, 3]); }, }; const e = createEmbedderWithProvider(cfg(), p); const controller = new AbortController(); - await e.embedOne("signal", { signal: controller.signal }); + const deadlineAt = Date.now() + 1_000; + await e.embedOne("signal", { signal: controller.signal, deadlineAt }); - expect(seen).toEqual([controller.signal]); + expect(seen).toEqual([{ signal: controller.signal, deadlineAt }]); }); it("dedups identical inputs into one provider call", async () => { @@ -192,6 +195,45 @@ describe("embedder facade", () => { } }); + it("preserves deferred retry diagnostics in error and status sinks", async () => { + const errors: EmbeddingErrorDetail[] = []; + const statuses: EmbeddingStatusDetail[] = []; + const retryAt = Date.now() + 120_000; + const provider: EmbeddingProvider = { + name: "openai_compatible", + async embed() { + throw new MemosError("embedding_unavailable", "provider cooldown", { + retryAfterMs: 120_000, + retryAt, + retryDecision: "defer", + retryReason: "retry_after_too_long", + }); + }, + }; + const e = createEmbedderWithProvider( + cfg({ onError: (detail) => errors.push(detail), onStatus: (detail) => statuses.push(detail) }), + provider, + ); + + await expect(e.embedOne("x")).rejects.toBeInstanceOf(MemosError); + + expect(errors).toContainEqual( + expect.objectContaining({ + retryAfterMs: 120_000, + retryAt, + retryDecision: "defer", + retryReason: "retry_after_too_long", + }), + ); + expect(statuses).toContainEqual( + expect.objectContaining({ + status: "error", + retryAt, + retryDecision: "defer", + }), + ); + }); + it("rejects when provider returns too few rows", async () => { const e = createEmbedderWithProvider(cfg({ provider: "gemini" }), new WrongCountProvider()); await expect(e.embedMany(["x", "y", "z"])).rejects.toBeInstanceOf(MemosError); diff --git a/apps/memos-local-plugin/tests/unit/embedding/fetcher.test.ts b/apps/memos-local-plugin/tests/unit/embedding/fetcher.test.ts index 73f37fed4..2811e94ef 100644 --- a/apps/memos-local-plugin/tests/unit/embedding/fetcher.test.ts +++ b/apps/memos-local-plugin/tests/unit/embedding/fetcher.test.ts @@ -4,6 +4,7 @@ import { MemosError } from "../../../agent-contract/errors.js"; import { initTestLogger } from "../../../core/logger/index.js"; import { httpPostJson } from "../../../core/embedding/fetcher.js"; import type { ProviderLogger } from "../../../core/embedding/types.js"; +import { clearRetryCooldowns } from "../../../core/util/retry-after.js"; function nullLogger(): ProviderLogger { return { @@ -21,6 +22,7 @@ describe("embedding/fetcher", () => { vi.useRealTimers(); // retry backoff uses real setTimeout; keep it real but short }); afterEach(() => { + clearRetryCooldowns(); vi.useRealTimers(); vi.unstubAllGlobals(); }); @@ -108,6 +110,59 @@ describe("embedding/fetcher", () => { vi.useRealTimers(); }); + it("defers a long Retry-After and short-circuits the provider cooldown", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-08-04T00:00:00.000Z")); + const f = mockFetch([ + new Response("maintenance", { status: 503, headers: { "Retry-After": "120" } }), + ]); + const opts = { + url: "https://embedding-x", + body: {}, + provider: "cohere" as const, + log: nullLogger(), + maxRetries: 1, + }; + + await expect(httpPostJson(opts)).rejects.toMatchObject({ + code: "embedding_unavailable", + details: { + retryAfterMs: 120_000, + retryDecision: "defer", + retryReason: "retry_after_too_long", + }, + }); + await expect(httpPostJson(opts)).rejects.toMatchObject({ + code: "embedding_unavailable", + details: { retryReason: "cooldown_active" }, + }); + expect(f).toHaveBeenCalledTimes(1); + }); + + it("returns structured diagnostics when network backoff cannot fit the deadline", async () => { + vi.useFakeTimers(); + const now = Date.parse("2026-08-04T00:00:00.000Z"); + vi.setSystemTime(now); + const f = mockFetch([new Error("ECONNRESET")]); + + await expect(httpPostJson({ + url: "https://embedding-deadline", + body: {}, + provider: "mistral", + log: nullLogger(), + maxRetries: 1, + deadlineAt: now + 100, + })).rejects.toMatchObject({ + name: "MemosError", + code: "embedding_unavailable", + details: { + retryDecision: "defer", + retryReason: "deadline_insufficient", + }, + }); + expect(f).toHaveBeenCalledTimes(1); + }); + it("does not retry on 400", async () => { mockFetch([new Response("bad", { status: 400 })]); await expect( diff --git a/apps/memos-local-plugin/tests/unit/embedding/retry-worker.test.ts b/apps/memos-local-plugin/tests/unit/embedding/retry-worker.test.ts index 5d7cd99b9..ae72cc60e 100644 --- a/apps/memos-local-plugin/tests/unit/embedding/retry-worker.test.ts +++ b/apps/memos-local-plugin/tests/unit/embedding/retry-worker.test.ts @@ -1,6 +1,7 @@ import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { createEmbeddingRetryWorker } from "../../../core/embedding/retry-worker.js"; +import { ERROR_CODES, MemosError } from "../../../agent-contract/errors.js"; import { rootLogger } from "../../../core/logger/index.js"; import type { EpisodeId, SessionId, TraceId } from "../../../core/types.js"; import { makeTmpDb, type TmpDbHandle } from "../../helpers/tmp-db.js"; @@ -174,6 +175,39 @@ describe("embedding retry worker", () => { expect(handle.repos.apiLogs.list({ toolName: "system_error", limit: 5, offset: 0 })).toHaveLength(1); }); + it("never schedules a durable retry before the provider retryAt", async () => { + const retryAt = NOW + 120_000; + handle.repos.embeddingRetryQueue.enqueue({ + id: "er_retry_after", + targetKind: "trace", + targetId: "tr_retry", + vectorField: "vec_summary", + sourceText: "retry me later", + maxAttempts: 3, + now: NOW, + }); + const worker = createEmbeddingRetryWorker({ + repos: handle.repos, + embedder: fakeEmbedder({ + throwWith: new MemosError( + ERROR_CODES.EMBEDDING_UNAVAILABLE, + "provider cooling down", + { retryAt, retryAfterMs: 120_000, retryDecision: "defer" }, + ), + }), + log: rootLogger.child({ channel: "test.embedding.retry" }), + now: () => NOW, + }); + + await worker.flush(); + + expect(queueRow(handle, "er_retry_after")).toMatchObject({ + status: "pending", + attempts: 1, + next_attempt_at: retryAt, + }); + }); + it("treats missing target rows as retry failures", async () => { handle.repos.embeddingRetryQueue.enqueue({ id: "er_missing", diff --git a/apps/memos-local-plugin/tests/unit/llm/client.test.ts b/apps/memos-local-plugin/tests/unit/llm/client.test.ts index dee0de228..cd125ecd8 100644 --- a/apps/memos-local-plugin/tests/unit/llm/client.test.ts +++ b/apps/memos-local-plugin/tests/unit/llm/client.test.ts @@ -283,6 +283,45 @@ describe("llm/client", () => { await expect(client.complete([] as LlmMessage[])).rejects.toBeInstanceOf(MemosError); }); + it("preserves deferred retry diagnostics in error and status sinks", async () => { + const errors: Array> = []; + const statuses: LlmStatusDetail[] = []; + const retryAt = Date.now() + 120_000; + const provider = new ThrowingProvider( + new MemosError(ERROR_CODES.LLM_RATE_LIMITED, "provider cooldown", { + retryAfterMs: 120_000, + retryAt, + retryDecision: "defer", + retryReason: "retry_after_too_long", + }), + ); + const client = createLlmClientWithProvider( + cfg({ + onError: (detail) => errors.push(detail as unknown as Record), + onStatus: (detail) => statuses.push(detail), + }), + provider, + ); + + await expect(client.complete("x")).rejects.toBeInstanceOf(MemosError); + + expect(errors).toContainEqual( + expect.objectContaining({ + retryAfterMs: 120_000, + retryAt, + retryDecision: "defer", + retryReason: "retry_after_too_long", + }), + ); + expect(statuses).toContainEqual( + expect.objectContaining({ + status: "error", + retryAt, + retryDecision: "defer", + }), + ); + }); + // ─── Circuit breaker (issue #1897) ────────────────────────────────────── describe("circuit breaker", () => { function statusSink(): { rows: LlmStatusDetail[]; push: (d: LlmStatusDetail) => void } { diff --git a/apps/memos-local-plugin/tests/unit/llm/fetcher.test.ts b/apps/memos-local-plugin/tests/unit/llm/fetcher.test.ts index 8bbc4ab7a..9b17e1b16 100644 --- a/apps/memos-local-plugin/tests/unit/llm/fetcher.test.ts +++ b/apps/memos-local-plugin/tests/unit/llm/fetcher.test.ts @@ -4,6 +4,7 @@ import { MemosError } from "../../../agent-contract/errors.js"; import { decodeSse, httpPostJson, httpPostStream } from "../../../core/llm/fetcher.js"; import { initTestLogger } from "../../../core/logger/index.js"; import type { LlmProviderLogger } from "../../../core/llm/types.js"; +import { clearRetryCooldowns } from "../../../core/util/retry-after.js"; function nullLog(): LlmProviderLogger { return { @@ -30,6 +31,7 @@ function mockFetch(replies: Array) { describe("llm/fetcher", () => { beforeAll(() => initTestLogger()); afterEach(() => { + clearRetryCooldowns(); vi.useRealTimers(); vi.unstubAllGlobals(); }); @@ -61,7 +63,7 @@ describe("llm/fetcher", () => { vi.useFakeTimers(); const ctrl = new AbortController(); const f = mockFetch([ - new Response("slow down", { status: 429, headers: { "Retry-After": "120" } }), + new Response("slow down", { status: 429, headers: { "Retry-After": "2" } }), ]); const pending = httpPostJson({ @@ -80,11 +82,12 @@ describe("llm/fetcher", () => { vi.useRealTimers(); }); - it("caps a long Retry-After from a 503 response", async () => { + it("defers a long Retry-After from a 503 without retrying early", async () => { vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-08-04T00:00:00.000Z")); + const warn = vi.fn(); const f = mockFetch([ new Response("maintenance", { status: 503, headers: { "Retry-After": "120" } }), - new Response(JSON.stringify({ ok: 1 }), { status: 200 }), ]); const pending = httpPostJson({ @@ -93,12 +96,137 @@ describe("llm/fetcher", () => { timeoutMs: 120_000, maxRetries: 1, provider: "openai_compatible", + log: { ...nullLog(), warn }, + }); + await expect(pending).rejects.toMatchObject({ + code: "llm_unavailable", + details: { + retryAfterMs: 120_000, + retryDecision: "defer", + retryReason: "retry_after_too_long", + }, + }); + expect(f).toHaveBeenCalledTimes(1); + expect(warn).toHaveBeenCalledWith( + "http.retry_deferred", + expect.objectContaining({ + retryAfterMs: 120_000, + retryDecision: "defer", + retryReason: "retry_after_too_long", + }), + ); + }); + + it("short-circuits calls while the provider Retry-After cooldown is active", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-08-04T00:00:00.000Z")); + const warn = vi.fn(); + const f = mockFetch([ + new Response("slow down", { status: 429, headers: { "Retry-After": "120" } }), + ]); + const opts = { + url: "https://x", + body: {}, + timeoutMs: 5_000, + maxRetries: 1, + provider: "openai_compatible" as const, + log: { ...nullLog(), warn }, + }; + + await expect(httpPostJson(opts)).rejects.toMatchObject({ code: "llm_rate_limited" }); + await expect(httpPostJson(opts)).rejects.toMatchObject({ + code: "llm_rate_limited", + details: { retryDecision: "defer", retryReason: "cooldown_active" }, + }); + expect(f).toHaveBeenCalledTimes(1); + expect(warn).toHaveBeenCalledWith( + "http.retry_cooldown", + expect.objectContaining({ retryReason: "cooldown_active" }), + ); + }); + + it("does not enter a Retry-After wait that cannot fit the absolute deadline", async () => { + vi.useFakeTimers(); + const now = Date.parse("2026-08-04T00:00:00.000Z"); + vi.setSystemTime(now); + const f = mockFetch([ + new Response("slow down", { status: 429, headers: { "Retry-After": "5" } }), + ]); + + await expect(httpPostJson({ + url: "https://deadline", + body: {}, + timeoutMs: 5_000, + maxRetries: 1, + deadlineAt: now + 1_000, + provider: "openai_compatible", log: nullLog(), + })).rejects.toMatchObject({ + code: "llm_rate_limited", + details: { + retryDecision: "defer", + retryReason: "deadline_insufficient", + }, }); - await vi.advanceTimersByTimeAsync(29_999); expect(f).toHaveBeenCalledTimes(1); - await vi.advanceTimersByTimeAsync(1); - await expect(pending).resolves.toMatchObject({ json: { ok: 1 } }); + }); + + it("returns structured diagnostics when network backoff cannot fit the deadline", async () => { + vi.useFakeTimers(); + const now = Date.parse("2026-08-04T00:00:00.000Z"); + vi.setSystemTime(now); + const f = mockFetch([new Error("ECONNRESET")]); + + await expect(httpPostJson({ + url: "https://network-deadline", + body: {}, + timeoutMs: 5_000, + maxRetries: 1, + deadlineAt: now + 100, + provider: "openai_compatible", + log: nullLog(), + })).rejects.toMatchObject({ + name: "MemosError", + code: "llm_unavailable", + details: { + retryDecision: "defer", + retryReason: "deadline_insufficient", + }, + }); + expect(f).toHaveBeenCalledTimes(1); + }); + + it("does not let an older in-flight success clear a newer provider cooldown", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-08-04T00:00:00.000Z")); + let resolveSuccess!: (response: Response) => void; + const success = new Promise((resolve) => { resolveSuccess = resolve; }); + const f = vi.fn() + .mockImplementationOnce(() => success) + .mockResolvedValueOnce( + new Response("slow down", { status: 429, headers: { "Retry-After": "120" } }), + ); + vi.stubGlobal("fetch", f); + const opts = { + url: "https://shared-endpoint", + body: {}, + timeoutMs: 5_000, + maxRetries: 1, + provider: "openai_compatible" as const, + log: nullLog(), + }; + + const older = httpPostJson<{ ok: boolean }>(opts); + await vi.waitFor(() => expect(f).toHaveBeenCalledTimes(1)); + await expect(httpPostJson(opts)).rejects.toMatchObject({ + details: { retryReason: "retry_after_too_long" }, + }); + resolveSuccess(new Response(JSON.stringify({ ok: true }), { status: 200 })); + await expect(older).resolves.toMatchObject({ json: { ok: true } }); + + await expect(httpPostJson(opts)).rejects.toMatchObject({ + details: { retryReason: "cooldown_active" }, + }); expect(f).toHaveBeenCalledTimes(2); }); diff --git a/apps/memos-local-plugin/tests/unit/util/retry-after.test.ts b/apps/memos-local-plugin/tests/unit/util/retry-after.test.ts index 0d412939b..84a4c8ea4 100644 --- a/apps/memos-local-plugin/tests/unit/util/retry-after.test.ts +++ b/apps/memos-local-plugin/tests/unit/util/retry-after.test.ts @@ -1,6 +1,13 @@ import { describe, expect, it } from "vitest"; -import { parseRetryAfterMs, retryDelayMs } from "../../../core/util/retry-after.js"; +import { + clearRetryCooldowns, + getRetryCooldown, + parseRetryAfterMs, + planRetry, + recordRetryCooldown, + retryCooldownKey, +} from "../../../core/util/retry-after.js"; describe("parseRetryAfterMs", () => { it("parses delay-seconds", () => { @@ -22,13 +29,62 @@ describe("parseRetryAfterMs", () => { expect(parseRetryAfterMs(null, now)).toBeNull(); }); - it("caps a provider supplied Retry-After before scheduling a timer", () => { - expect(retryDelayMs({ + it("defers instead of retrying before a long provider Retry-After", () => { + expect(planRetry({ attempt: 1, baseMs: 200, jitterMaxMs: 0, retryAfterMs: 120_000, - maxDelayMs: 30_000, - })).toBe(30_000); + maxInlineDelayMs: 30_000, + nowMs: 1_000, + })).toEqual({ + action: "defer", + backoffMs: 200, + delayMs: 120_000, + reason: "retry_after_too_long", + retryAfterMs: 120_000, + retryAt: 121_000, + source: "retry_after", + }); + }); + + it("defers when an otherwise short retry cannot fit the request deadline", () => { + expect(planRetry({ + attempt: 1, + baseMs: 200, + jitterMaxMs: 0, + retryAfterMs: 2_000, + deadlineAt: 2_500, + nowMs: 1_000, + })).toMatchObject({ + action: "defer", + reason: "deadline_insufficient", + retryAt: 3_000, + }); + }); + + it("keeps provider cooldowns monotonic and expires them at retryAt", () => { + clearRetryCooldowns(); + recordRetryCooldown("llm:test", { + retryAfterMs: 2_000, + retryAt: 3_000, + status: 429, + }); + recordRetryCooldown("llm:test", { + retryAfterMs: 500, + retryAt: 1_500, + status: 503, + }); + expect(getRetryCooldown("llm:test", 2_999)).toMatchObject({ + retryAt: 3_000, + status: 429, + }); + expect(getRetryCooldown("llm:test", 3_000)).toBeNull(); + clearRetryCooldowns(); + }); + + it("scopes provider cooldowns by endpoint and model", () => { + expect(retryCooldownKey("llm", "openai_compatible", "https://x", "model-a")) + .not.toBe(retryCooldownKey("llm", "openai_compatible", "https://x", "model-b")); }); }); From 28e454dcb543c9431c571d56c13483589a965f2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=81=E5=9C=A8=E5=90=B5=E7=9D=80=E5=90=83=E7=B3=96?= Date: Tue, 4 Aug 2026 19:56:39 +0800 Subject: [PATCH 17/18] fix(plugin): show memory add roles correctly --- .../core/pipeline/memory-core.ts | 88 +++++++++++-------- .../tests/unit/pipeline/memory-core.test.ts | 47 ++++++++++ 2 files changed, 100 insertions(+), 35 deletions(-) diff --git a/apps/memos-local-plugin/core/pipeline/memory-core.ts b/apps/memos-local-plugin/core/pipeline/memory-core.ts index c9254e092..a0354bf64 100644 --- a/apps/memos-local-plugin/core/pipeline/memory-core.ts +++ b/apps/memos-local-plugin/core/pipeline/memory-core.ts @@ -1232,18 +1232,59 @@ export function createMemoryCore( const statsLine = `phase=${phase}, stored=${storedCount}` + (r.warnings.length > 0 ? `, warnings=${r.warnings.length}` : ""); - const details = r.traces.map((tc) => ({ - role: inferTurnRole(tc), - action: phase === "lite" ? ("stored" as const) : ("reflected" as const), - summary: tc.reflection?.text ?? null, - content: ( - tc.userText || - tc.agentText || - summarizeToolCalls(tc.toolCalls) || - "" - ).slice(0, 400), - traceId: tc.traceId, - })); + const action = phase === "lite" + ? ("stored" as const) + : ("reflected" as const); + const details = r.traces.flatMap((tc) => { + const items: Array<{ + role: "user" | "assistant" | "tool" | "reflection" | "other"; + action: typeof action; + summary: string | null; + content: string; + traceId: string; + }> = []; + + if (tc.userText) { + items.push({ + role: "user", + action, + summary: null, + content: tc.userText.slice(0, 400), + traceId: tc.traceId, + }); + } + if (tc.agentText) { + items.push({ + role: "assistant", + action, + summary: null, + content: tc.agentText.slice(0, 400), + traceId: tc.traceId, + }); + } + + const toolSummary = summarizeToolCalls(tc.toolCalls); + if (items.length === 0) { + items.push({ + role: toolSummary ? "tool" : "other", + action, + summary: tc.reflection?.text ?? null, + content: toolSummary.slice(0, 400), + traceId: tc.traceId, + }); + } else if (tc.reflection?.text) { + // Keep the existing reflect-phase summary visible without + // presenting it as either side's original chat content. + items.push({ + role: "reflection", + action, + summary: tc.reflection.text, + content: "", + traceId: tc.traceId, + }); + } + return items; + }); handle.repos.apiLogs.insert({ toolName: "memory_add", input: { @@ -6195,26 +6236,3 @@ function summarizeToolCalls( }) .join("\n"); } - -/** - * Heuristic role inference for api_logs "memory_add" rows — mirrors - * the legacy plugin's behaviour where each captured turn showed up - * labelled `user` / `assistant` / `tool` on the Logs page. - * - * Priority: if the step carries userText (the user's query), label it - * "user" even when toolCalls are present — this is the first sub-step - * of a multi-tool turn and semantically represents the user request. - */ -function inferTurnRole(step: { - userText?: string; - agentText?: string; - toolCalls?: readonly unknown[]; -}): "user" | "assistant" | "tool" | "other" { - const u = (step.userText ?? "").length; - const a = (step.agentText ?? "").length; - if (u > 0 && (step.toolCalls?.length ?? 0) > 0) return "user"; - if ((step.toolCalls?.length ?? 0) > 0) return "tool"; - if (u >= a && u > 0) return "user"; - if (a > 0) return "assistant"; - return "other"; -} diff --git a/apps/memos-local-plugin/tests/unit/pipeline/memory-core.test.ts b/apps/memos-local-plugin/tests/unit/pipeline/memory-core.test.ts index cb5140d09..52853ff4a 100644 --- a/apps/memos-local-plugin/tests/unit/pipeline/memory-core.test.ts +++ b/apps/memos-local-plugin/tests/unit/pipeline/memory-core.test.ts @@ -898,6 +898,53 @@ describe("MemoryCore façade", () => { expect(scored.priority).toBe(1); }); + it("logs both user and assistant content with the matching role", async () => { + pipeline = createPipeline(buildDeps(db!)); + core = createMemoryCore( + pipeline, + resolveHome("openclaw", "/tmp/memos-mc-test"), + "test", + ); + await core.init(); + + const userText = "今晚吃什么,推荐一下"; + const agentText = "可以考虑清淡的汤面、盖饭或者附近评价不错的家常菜。"; + const start = await core.onTurnStart({ + agent: "openclaw", + sessionId: "s-memory-add-roles", + userText, + ts: 1_700_000_000_000, + }); + await core.onTurnEnd({ + agent: "openclaw", + sessionId: start.query.sessionId!, + episodeId: start.query.episodeId!, + agentText, + toolCalls: [], + ts: 1_700_000_000_500, + }); + + const { logs } = await core.listApiLogs({ + toolName: "memory_add", + limit: 10, + }); + const liteLog = logs.find((log) => { + const input = JSON.parse(log.inputJson) as { phase?: string }; + return input.phase === "lite"; + }); + expect(liteLog).toBeDefined(); + + const output = JSON.parse(liteLog!.outputJson) as { + details?: Array<{ role?: string; content?: string }>; + }; + expect( + output.details?.map(({ role, content }) => ({ role, content })), + ).toEqual([ + { role: "user", content: userText }, + { role: "assistant", content: agentText }, + ]); + }); + it("onTurnEnd preserves adapter-provided historical timestamps", async () => { pipeline = createPipeline(buildDeps(db!)); core = createMemoryCore( From 1669c660f30dc5ce4cb8a5511c3978bcfc24bd4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=81=E5=9C=A8=E5=90=B5=E7=9D=80=E5=90=83=E7=B3=96?= Date: Tue, 4 Aug 2026 20:04:47 +0800 Subject: [PATCH 18/18] style(plugin): format Hermes bridge changes --- .../adapters/hermes/memos_provider/__init__.py | 5 ++--- .../adapters/hermes/memos_provider/bridge_client.py | 6 +++--- .../memos-local-plugin/tests/python/test_bridge_client.py | 8 ++------ 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py b/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py index 6353c560a..79b88cbf4 100644 --- a/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py +++ b/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py @@ -241,6 +241,7 @@ def _remaining_rpc_timeout( return remaining return min(requested_timeout, remaining) + _HERMES_INTERNAL_REVIEW_PREFIXES = ( "review the conversation above and consider saving to memory if appropriate.", "review the conversation above and update the skill library.", @@ -2108,9 +2109,7 @@ def _bridge_request_with_retry( err, ) reconnect_ceiling = ( - timeout - if deadline_monotonic is not None and timeout is not None - else 30.0 + timeout if deadline_monotonic is not None and timeout is not None else 30.0 ) reconnect_timeout = _remaining_rpc_timeout( deadline_monotonic, diff --git a/apps/memos-local-plugin/adapters/hermes/memos_provider/bridge_client.py b/apps/memos-local-plugin/adapters/hermes/memos_provider/bridge_client.py index 016c93ae5..cbb2fa84c 100644 --- a/apps/memos-local-plugin/adapters/hermes/memos_provider/bridge_client.py +++ b/apps/memos-local-plugin/adapters/hermes/memos_provider/bridge_client.py @@ -160,9 +160,9 @@ def __init__( # demultiplexing for every shared provider lease. self._host_handlers: dict[str, Callable[[dict[str, Any]], Any]] = {} self._host_handlers_cv = threading.Condition() - self._host_handler_queue: queue.Queue[ - tuple[Any, str, dict[str, Any]] | None - ] = queue.Queue(maxsize=HOST_HANDLER_QUEUE_CAPACITY) + self._host_handler_queue: queue.Queue[tuple[Any, str, dict[str, Any]] | None] = queue.Queue( + maxsize=HOST_HANDLER_QUEUE_CAPACITY + ) self._host_handler_stop = threading.Event() self._closed = False diff --git a/apps/memos-local-plugin/tests/python/test_bridge_client.py b/apps/memos-local-plugin/tests/python/test_bridge_client.py index 1bd9dce3f..b5c9eb917 100644 --- a/apps/memos-local-plugin/tests/python/test_bridge_client.py +++ b/apps/memos-local-plugin/tests/python/test_bridge_client.py @@ -491,9 +491,7 @@ def _slow_handler(_params: dict) -> dict: finally: release_handler.set() - reverse_response = self._wait_for_client_write( - lambda msg: msg.get("id") == "srv-slow" - ) + reverse_response = self._wait_for_client_write(lambda msg: msg.get("id") == "srv-slow") self.assertEqual(reverse_response["result"]["text"], "host:done") client.close() @@ -536,9 +534,7 @@ def _slow_handler(_params: dict) -> dict: ) try: - response = self._wait_for_client_write( - lambda msg: msg.get("id") == overflow_id - ) + response = self._wait_for_client_write(lambda msg: msg.get("id") == overflow_id) self.assertEqual(response["error"]["data"]["code"], "host_handler_busy") finally: client.close()