diff --git a/justfile b/justfile index 276a86066..95e927017 100644 --- a/justfile +++ b/justfile @@ -575,10 +575,11 @@ migration message: set-version version scope="all": python3 scripts/update_versions.py "{{version}}" --scope "{{scope}}" -# Pin the Basic Memory Git ref used by both Codex uv hook scripts. +# Pin the Basic Memory Git ref used by all Codex uv hook scripts. set-codex-hook-version ref: uv add --script plugins/codex/hooks/session_start.py --raw "basic-memory @ git+https://github.com/basicmachines-co/basic-memory@{{ref}}" uv add --script plugins/codex/hooks/pre_compact.py --raw "basic-memory @ git+https://github.com/basicmachines-co/basic-memory@{{ref}}" + uv add --script plugins/codex/hooks/stop.py --raw "basic-memory @ git+https://github.com/basicmachines-co/basic-memory@{{ref}}" # Preview a version update without writing (scope: all | core | packages) set-version-dry-run version scope="all": @@ -664,6 +665,7 @@ release version: plugins/codex/.codex-plugin/plugin.json \ plugins/codex/hooks/session_start.py \ plugins/codex/hooks/pre_compact.py \ + plugins/codex/hooks/stop.py \ integrations/hermes/plugin.yaml \ integrations/hermes/__init__.py \ integrations/openclaw/package.json @@ -784,6 +786,7 @@ beta version: plugins/codex/.codex-plugin/plugin.json \ plugins/codex/hooks/session_start.py \ plugins/codex/hooks/pre_compact.py \ + plugins/codex/hooks/stop.py \ integrations/hermes/plugin.yaml \ integrations/hermes/__init__.py \ integrations/openclaw/package.json diff --git a/plugins/claude-code/README.md b/plugins/claude-code/README.md index 8ba8b1bd5..b8022825c 100644 --- a/plugins/claude-code/README.md +++ b/plugins/claude-code/README.md @@ -94,7 +94,8 @@ local checkout. Two disclosures: - **Event capture is opt-in and off by default.** Setting `captureEvents: true` (the JSON boolean — strings never enable it) records redacted lifecycle-event envelopes to a local inbox under your Basic Memory home. Inspect with - `basic-memory hook status`, project with `basic-memory hook flush`. + `basic-memory hook status`, archive locally with `basic-memory hook flush`. + The lifecycle trace never becomes a graph note. Every failure path exits 0 — the hooks stay invisible rather than disrupt a session. @@ -160,10 +161,9 @@ general setup seeds `session`; a **coding setup** (persisted as `sessionProfile: "coding"` with a user-confirmed `repository`) seeds `coding_session`, whose required repository, repo-root, working-directory, branch, and Git SHA frontmatter make checkpoints queryable by structured -filters; typed pull-request fields are added when a PR exists. Optional flush projection also writes -normalized `session` and `tool_ledger` artifacts. Those projection contracts are -owned and tested by Basic Memory core rather than copied into separate -host-plugin schemas. +filters; typed pull-request fields are added when a PR exists. Lifecycle-event +envelopes are operational trace rather than knowledge; `bm hook flush` archives +them locally and never creates session or tool-ledger notes. To customize how Claude writes memory, edit `skills/bm-writing/SKILL.md` in the plugin source. `bm-remember` and the output style's capture reflexes apply that diff --git a/plugins/claude-code/skills/bm-orient/SKILL.md b/plugins/claude-code/skills/bm-orient/SKILL.md index 21ea169e0..f35677ca1 100644 --- a/plugins/claude-code/skills/bm-orient/SKILL.md +++ b/plugins/claude-code/skills/bm-orient/SKILL.md @@ -36,8 +36,9 @@ of a session; this is the deliberate mid-session version with deeper reads. report that setup is incomplete. Merge and deduplicate the results, sort them newest first, and prefer the highest-signal checkpoint regardless of which producer wrote it. `coding_session` carries schema-required, queryable Git - context; `session` covers general checkpoints, PreCompact captures, and - normalized `bm hook flush` projections. + context; `session` covers general checkpoints and PreCompact captures. Do not + query lifecycle trace: `bm hook flush` archives it locally rather than + promoting it into the graph. 3. Query configured `secondaryProjects` read-only for open decisions. Do not write to shared projects during orientation. diff --git a/plugins/claude-code/skills/bm-setup/SKILL.md b/plugins/claude-code/skills/bm-setup/SKILL.md index 76b17be67..5bc3c57e4 100644 --- a/plugins/claude-code/skills/bm-setup/SKILL.md +++ b/plugins/claude-code/skills/bm-setup/SKILL.md @@ -112,10 +112,11 @@ Ask only what you can't infer. Cover: schemas" below.) 6. **Lifecycle-event capture.** "Should I also keep a local, redacted trail of - SessionStart and PreCompact events for later projection?" Default to **off**. + SessionStart and PreCompact events for diagnostics?" Default to **off**. Explain that the normal session brief and PreCompact checkpoint work either way; enabling this adds envelopes to a local inbox until `bm hook flush` - projects them. Only the JSON boolean `true` enables capture. + archives them locally. It never creates knowledge-graph notes. Only the JSON + boolean `true` enables capture. - If enabled, optionally ask for repo-specific `redactKeys` (additional payload keys) and `redactPaths` (working directories or path-bearing content). The built-in redaction floor still applies when these lists are empty. @@ -144,10 +145,9 @@ from this skill's directory, then `schemas/`** (this skill is at `/skills/bm-setup/`). Read `coding-session.md` for a coding profile or `session.md` for a general profile, then read `decision.md` and `task.md`. -These schemas cover notes the Claude integration writes directly. The normalized -`session` and `tool_ledger` artifacts written by `bm hook flush` are core-owned -projections; their shape belongs to the core projector and its tests, not a -duplicated per-plugin `tool_ledger` schema. +These schemas cover notes the Claude integration writes directly. Lifecycle +envelopes are not notes: `bm hook flush` archives that operational trace locally, +so there is no projected session or tool-ledger schema to seed. For each one: - Check whether the chosen project already has a schema for that type diff --git a/plugins/claude-code/skills/bm-status/SKILL.md b/plugins/claude-code/skills/bm-status/SKILL.md index eeefebbdc..64ad9a736 100644 --- a/plugins/claude-code/skills/bm-status/SKILL.md +++ b/plugins/claude-code/skills/bm-status/SKILL.md @@ -32,7 +32,8 @@ This is a quick diagnostic — gather the facts and lay them out; don't over-inv 3. **Core hook health.** With the first available launcher, run `basic-memory hook status --harness claude --project-dir `. - Report its shared inbox path, pending envelopes, processed envelopes, last + Report its shared inbox path, pending envelopes, archived envelopes, pending + checkpoint requests, last flush, settings state, resolved primary project, capture state, capture folder, Basic Memory version, and uv version. Inbox counts are global across supported harnesses; do not attribute a backlog solely to Claude. Treat this command's @@ -76,7 +77,8 @@ you couldn't determine, rather than failing the whole report): - Redact paths: - Shared hook inbox: - Shared pending envelopes: -- Shared processed envelopes: +- Shared archived envelopes: +- Pending checkpoint requests: - Last flush: - Hook runtime: basic-memory ; uv - Recent checkpoints: diff --git a/plugins/codex/DEVELOPMENT.md b/plugins/codex/DEVELOPMENT.md index 275a19bb1..4aaa99131 100644 --- a/plugins/codex/DEVELOPMENT.md +++ b/plugins/codex/DEVELOPMENT.md @@ -38,7 +38,7 @@ projects on the same machine. Repo-specific memory routing still comes from each ## Iteration Loop -Pin both hook scripts to the Basic Memory revision under test: +Pin all three hook scripts to the Basic Memory revision under test: ```bash just set-codex-hook-version "$(git rev-parse origin/main)" diff --git a/plugins/codex/README.md b/plugins/codex/README.md index 8d51a9836..06fc21400 100644 --- a/plugins/codex/README.md +++ b/plugins/codex/README.md @@ -11,9 +11,10 @@ verification, decision capture, and resumable checkpoints. - **Orient from memory.** The `bm-orient` skill reads active tasks, open decisions, and recent Codex checkpoints before substantial work. -- **Checkpoint work.** The `bm-checkpoint` skill and `PreCompact` hook write - general `codex_session` notes or schema-backed `coding_session` notes with - structured repository and pull-request context. +- **Checkpoint work.** `PreCompact` records a private request and the `Stop` + hook asks the active Codex turn to run `bm-checkpoint` once after compaction. + The resulting `codex_session` or `coding_session` note is agent-authored from + the compacted working context, with repository and pull-request evidence. - **Capture decisions.** The `bm-decide` skill records durable engineering decisions with rationale, alternatives, and consequences. - **Remember lightly.** The `bm-remember` skill saves small facts without turning @@ -32,16 +33,18 @@ verification, decision capture, and resumable checkpoints. | --- | --- | | `.codex-plugin/plugin.json` | Codex plugin manifest | | `.mcp.json` | Basic Memory MCP server configuration | -| `hooks/hooks.json` | SessionStart and PreCompact hook registration | +| `hooks/hooks.json` | SessionStart, PreCompact, and Stop hook registration | | `hooks/session_start.py` | uv script: runs `basic-memory hook session-start --harness codex` | | `hooks/pre_compact.py` | uv script: runs `basic-memory hook pre-compact --harness codex` | +| `hooks/stop.py` | uv script: runs `basic-memory hook stop --harness codex` | | `skills/` | Codex-native Basic Memory workflows | | `schemas/` | Seed schemas for Codex sessions, decisions, and tasks | -The hook scripts carry no logic: the brief, the checkpoint, and lifecycle-event -capture all live in the pinned Basic Memory revision behind `bm hook`. Each is -a self-contained PEP 723 script pinned to a Basic Memory Git ref. Both refs -are updated together with `just set-codex-hook-version `. +The hook scripts carry no logic: the brief, checkpoint coordination, and +lifecycle-event capture all live in the pinned Basic Memory revision behind +`bm hook`. Each is a self-contained PEP 723 script pinned to a Basic Memory Git +ref. All refs are updated together with +`just set-codex-hook-version `. ## Requirements @@ -69,6 +72,13 @@ Plugin installation is user-level in Codex, so one install makes the plugin available across projects on the same machine. Start a new Codex thread after installing so Codex can load the plugin skills, MCP configuration, and hooks. +When adding the marketplace from the Git repository UI, leave **Sparse paths** +empty. If a sparse checkout is required, include both `.agents/plugins` and +`plugins/codex`. Selecting only `plugins/codex` omits +`.agents/plugins/marketplace.json`, so Codex correctly reports that the checked +out marketplace root has no supported manifest. The marketplace file should not +be moved into the plugin directory. + Configuration can live at user level in `~/.codex/basic-memory.json` or at project level in `.codex/basic-memory.json`. User-level settings are the base; the nearest project file overrides only the keys it declares. `redactKeys` and @@ -103,9 +113,16 @@ Run the setup skill, or create `~/.codex/basic-memory.json` for shared defaults: Codex event capture is on by default. Set the JSON boolean `false` at user or project level to opt out; malformed values fail closed. Captured, redacted -lifecycle-event envelopes land in a local inbox under your Basic Memory home -(`basic-memory hook status` / `basic-memory hook flush`). Add `redactKeys` and -`redactPaths` arrays to extend the built-in redaction floor. +lifecycle-event envelopes land in a local inbox under your Basic Memory home. +The lifecycle trace stays local: `basic-memory hook flush` only moves valid +envelopes into the local retention archive and never creates graph notes. Add +`redactKeys` and `redactPaths` arrays to extend the built-in redaction floor. + +Codex ignores PreCompact stdout, so PreCompact cannot ask the model to write a +note directly. It leaves a private request for the Stop hook. Stop then blocks +the turn once with a request to run `bm-checkpoint`; the active model writes an +agent-authored checkpoint from its compacted context, and the next Stop is a +no-op to prevent loops. When `captureFolder` is omitted, Codex resolves the Git top-level directory and writes to `codex/`. An explicit folder still wins. @@ -125,12 +142,9 @@ identifier in the project file without duplicating the shared settings: The plugin's seed schemas cover notes Codex writes directly: `codex_session`, `coding_session`, `decision`, and `task`. Coding sessions require structured repository, repository-root, working-directory, branch, and Git SHA frontmatter; -current pull-request fields are added when a PR exists. Optional flush projection -also writes normalized `session` and `tool_ledger` artifacts. Those are -core-owned contracts implemented and tested with the projector, not duplicate -schema files maintained by each host plugin. `bm-orient` and `bm-status` still -recall normalized `session` notes -alongside Codex checkpoints. +current pull-request fields are added when a PR exists. Lifecycle envelopes are +operational trace rather than knowledge, so orientation only recalls authored +checkpoint types. Codex plugin hooks must be reviewed and trusted before they run. Open `/hooks` in Codex after enabling the plugin and trust the Basic Memory hook definitions. diff --git a/plugins/codex/hooks/hooks.json b/plugins/codex/hooks/hooks.json index f379e7754..352888501 100644 --- a/plugins/codex/hooks/hooks.json +++ b/plugins/codex/hooks/hooks.json @@ -1,5 +1,5 @@ { - "description": "Basic Memory for Codex hooks - orient from the graph on session start and checkpoint before compaction.", + "description": "Basic Memory for Codex hooks - orient from the graph, record local lifecycle trace, and request an authored checkpoint after compaction.", "hooks": { "SessionStart": [ { @@ -26,6 +26,18 @@ } ] } + ], + "Stop": [ + { + "hooks": [ + { + "type": "command", + "command": "uv run --quiet --script \"${PLUGIN_ROOT}/hooks/stop.py\"", + "statusMessage": "Finishing the Basic Memory checkpoint", + "timeout": 30 + } + ] + } ] } } diff --git a/plugins/codex/hooks/pre_compact.py b/plugins/codex/hooks/pre_compact.py index f5cdd90cb..d8320948b 100755 --- a/plugins/codex/hooks/pre_compact.py +++ b/plugins/codex/hooks/pre_compact.py @@ -2,7 +2,7 @@ # /// script # requires-python = ">=3.12" # dependencies = [ -# "basic-memory @ git+https://github.com/basicmachines-co/basic-memory@bd5d145d5be3bab3c73291ae2a698f8e5a1e54cb", +# "basic-memory @ git+https://github.com/basicmachines-co/basic-memory@1cb7a541fbb43c5a4f1b552c0123a1c55beb8dfb", # ] # /// """PreCompact hook launcher backed by a pinned Basic Memory revision. diff --git a/plugins/codex/hooks/session_start.py b/plugins/codex/hooks/session_start.py index 426bf4dc8..7de758853 100755 --- a/plugins/codex/hooks/session_start.py +++ b/plugins/codex/hooks/session_start.py @@ -2,7 +2,7 @@ # /// script # requires-python = ">=3.12" # dependencies = [ -# "basic-memory @ git+https://github.com/basicmachines-co/basic-memory@bd5d145d5be3bab3c73291ae2a698f8e5a1e54cb", +# "basic-memory @ git+https://github.com/basicmachines-co/basic-memory@1cb7a541fbb43c5a4f1b552c0123a1c55beb8dfb", # ] # /// """SessionStart hook launcher backed by a pinned Basic Memory revision. diff --git a/plugins/codex/hooks/stop.py b/plugins/codex/hooks/stop.py new file mode 100755 index 000000000..078a8ce31 --- /dev/null +++ b/plugins/codex/hooks/stop.py @@ -0,0 +1,37 @@ +#!/usr/bin/env -S uv run --quiet --script +# /// script +# requires-python = ">=3.12" +# dependencies = [ +# "basic-memory @ git+https://github.com/basicmachines-co/basic-memory@1cb7a541fbb43c5a4f1b552c0123a1c55beb8dfb", +# ] +# /// +"""Stop hook launcher backed by a pinned Basic Memory revision. + +Stop must always return valid JSON. Running Typer without Click's standalone +mode avoids treating its normal completion as an exception; real failures emit +a fail-open response so Basic Memory can never strand a Codex turn. +""" + +import sys + +VERB = "stop" +HARNESS = "codex" + + +def hook_args() -> list[str]: + return ["hook", VERB, "--harness", HARNESS] + + +def main() -> None: + from basic_memory.cli.main import app + + sys.argv = ["basic-memory", *hook_args()] + app(standalone_mode=False) + + +if __name__ == "__main__": + try: + main() + except BaseException: # noqa: BLE001 - the documented fail-open boundary + print('{"continue":true}') + sys.exit(0) diff --git a/plugins/codex/hooks/test_codex_stop.py b/plugins/codex/hooks/test_codex_stop.py new file mode 100644 index 000000000..7c26eacdf --- /dev/null +++ b/plugins/codex/hooks/test_codex_stop.py @@ -0,0 +1,49 @@ +"""Tests for the Codex Stop uv hook script.""" + +import json +import os +import re +import subprocess +import sys +from pathlib import Path + + +SCRIPT = Path(__file__).with_name("stop.py") +GIT_DEPENDENCY_RE = re.compile( + r'"basic-memory @ git\+https://github\.com/' + r'basicmachines-co/basic-memory@([^"]+)"' +) + + +def test_script_fails_open_with_valid_json(tmp_path: Path) -> None: + env = os.environ.copy() + env["HOME"] = str(tmp_path) + env["USERPROFILE"] = str(tmp_path) + env["BASIC_MEMORY_CONFIG_DIR"] = str(tmp_path / "basic-memory") + + result = subprocess.run( + [sys.executable, str(SCRIPT)], + input='{"session_id":"none","stop_hook_active":false}', + env=env, + capture_output=True, + text=True, + timeout=60, + ) + + assert result.returncode == 0 + assert json.loads(result.stdout) == {"continue": True} + + +def test_dependency_is_pinned_to_a_basic_memory_git_ref() -> None: + text = SCRIPT.read_text(encoding="utf-8") + + assert len(GIT_DEPENDENCY_RE.findall(text)) == 1 + + +def test_metadata_and_command_shape() -> None: + text = SCRIPT.read_text(encoding="utf-8") + + assert text.count("# /// script") == 1 + assert re.search(r'^# requires-python = ">=3\.12"$', text, re.MULTILINE) + assert 'VERB = "stop"' in text + assert 'HARNESS = "codex"' in text diff --git a/plugins/codex/skills/bm-checkpoint/SKILL.md b/plugins/codex/skills/bm-checkpoint/SKILL.md index 246b744a7..2e5f49517 100644 --- a/plugins/codex/skills/bm-checkpoint/SKILL.md +++ b/plugins/codex/skills/bm-checkpoint/SKILL.md @@ -6,19 +6,23 @@ description: Save a deliberate Codex work checkpoint to Basic Memory with change # Checkpoint Codex Work Create a durable handoff note for current Codex work. Use this when the user asks -to checkpoint, wrap up, hand off, remember the state of the work, or before a long -context transition. +to checkpoint, wrap up, hand off, remember the state of the work, or when the +Stop hook requests the deliberate handoff after compaction. ## Gather Read `~/.codex/basic-memory.json`, then the nearest project -`.codex/basic-memory.json`; project keys override user keys: +`.codex/basic-memory.json`; project keys override user keys except that +`redactKeys` and `redactPaths` accumulate so project config cannot weaken user +privacy: - `primaryProject`, default omitted - `captureFolder`, default `codex/` - `placementConventions`, optional - `sessionProfile`, default `general` - `repository`, required when `sessionProfile` is `coding` +- `redactKeys`, optional additional secret field names +- `redactPaths`, optional additional private directory prefixes Apply the `bm-writing` skill before drafting the note. @@ -43,6 +47,31 @@ Gather repo evidence: Do not claim a test passed unless you ran it or the user supplied the result. +## Privacy Gate + +Apply this gate to deliberate checkpoints and Stop-requested checkpoints alike. +It is mandatory before any `write_note` call: + +1. Merge user and project `redactKeys` and `redactPaths` by accumulation. Include + the built-in secret-key families (`secret`, `token`, `password`, `credential`, + `auth`, API/access/private keys) and private roots (`~/.ssh`, `~/.aws`, and + `~/.gnupg`). Never copy the redaction rules themselves into the note. +2. Expand `~` for comparison and normalize both slash styles. A denied path + matches its exact directory or a descendant, not a sibling that merely shares + its text prefix. +3. Scrub **every string** passed to `write_note`: title, directory, tags, + frontmatter, body, observations, relations, changed-file paths, command + output, repository evidence, and pull-request evidence. Replace a whole path + value under a denied root, or an embedded denied-path token, with + `[REDACTED_PATH]`. Do not omit schema-required path fields; use the marker. +4. When gathered structured evidence has a key matching a built-in or configured + `redactKeys` entry, replace its entire value with `[REDACTED]`. Never reproduce + credentials, tokens, or private-key material from config, environment, tool + output, or compacted conversation context. +5. Review the final note arguments once more before writing. If any value cannot + be confidently scrubbed, fail closed: skip the checkpoint and report that + privacy policy blocked the write. Do not fall back to an unredacted note. + ## Write A checkpoint is a durable handoff, not a status dump or commit-by-commit diff --git a/plugins/codex/skills/bm-orient/SKILL.md b/plugins/codex/skills/bm-orient/SKILL.md index 393745912..dd87ee718 100644 --- a/plugins/codex/skills/bm-orient/SKILL.md +++ b/plugins/codex/skills/bm-orient/SKILL.md @@ -23,16 +23,16 @@ the user asks where things stand. - recent coding sessions: `type=coding_session`, `repository=`, after `recallTimeframe`, when `sessionProfile=coding` - - recent core-projected sessions: `type=session`, after `recallTimeframe` - Always query `codex_session` and `session`; include `coding_session` for a + Always query `codex_session`; include `coding_session` for a coding profile only with the configured `repository` metadata filter. Never run an unscoped coding-session query; if the repository is missing, report that setup is incomplete. Merge and deduplicate the results, sort them newest first, and prefer the highest-signal checkpoint regardless of which producer wrote it. `coding_session` carries schema-required, queryable Git - context; `codex_session` preserves general and legacy Codex checkpoints; - `session` carries normalized artifacts from `bm hook flush`. + context; `codex_session` preserves general and legacy Codex checkpoints. + Do not query lifecycle trace: `bm hook flush` archives it locally and never + promotes it into the graph. 3. Query configured `secondaryProjects` read-only for open decisions. Do not write to shared projects during orientation. diff --git a/plugins/codex/skills/bm-setup/SKILL.md b/plugins/codex/skills/bm-setup/SKILL.md index 4f4b8b191..039efc98a 100644 --- a/plugins/codex/skills/bm-setup/SKILL.md +++ b/plugins/codex/skills/bm-setup/SKILL.md @@ -58,8 +58,8 @@ coding checkpoints store structured repository, branch, SHA, working-directory, and optional pull-request metadata in Basic Memory. Explain the capture tradeoff before asking: enabled capture adds a local, -redacted event trail that stays queued until `bm hook flush` projects it. It does -not write to team projects. The default is enabled; an explicit JSON boolean +redacted event trail that stays queued until `bm hook flush` archives it locally. +It never creates knowledge-graph notes or writes to team projects. The default is enabled; an explicit JSON boolean `false` disables it, and malformed values fail closed. If there are duplicate names, show qualified names and ask the user which one to @@ -142,10 +142,9 @@ Then seed these schemas for both profiles: - `decision.md` - `task.md` -These schemas cover notes Codex writes directly. The normalized `session` and -`tool_ledger` artifacts written by `bm hook flush` are core-owned projections; -their shape belongs to the core projector and its tests, not duplicated plugin -schema files. They remain queryable by their `type` frontmatter. +These schemas cover notes Codex writes directly. Lifecycle envelopes are not +notes: `bm hook flush` archives that operational trace locally, so there is no +projected session or tool-ledger schema to seed. Use `write_note` with `directory="schemas"`, `note_type="schema"`, schema frontmatter as metadata, and the markdown body as content. Do not paste the YAML diff --git a/plugins/codex/skills/bm-status/SKILL.md b/plugins/codex/skills/bm-status/SKILL.md index a70eb5593..c3b23cf36 100644 --- a/plugins/codex/skills/bm-status/SKILL.md +++ b/plugins/codex/skills/bm-status/SKILL.md @@ -30,7 +30,8 @@ Gather a concise diagnostic. Do not over-investigate. 3. Core hook health: - with the first available launcher, run `basic-memory hook status --harness codex --project-dir ` - - report the shared inbox path, pending envelopes, processed envelopes, last + - report the shared inbox path, pending envelopes, archived envelopes, + pending checkpoint requests, last flush, settings state, resolved primary project, capture state, capture folder, Basic Memory version, and uv version from that command - inbox counts are global across supported harnesses; do not attribute a @@ -44,12 +45,12 @@ Gather a concise diagnostic. Do not over-investigate. they run 5. Basic Memory queries: - - query recent `type=codex_session` and `type=session`; when + - query recent `type=codex_session`; when `sessionProfile=coding`, also query `type=coding_session` with `repository=`, then merge, deduplicate, sort newest first, and keep the newest five; never run an unscoped coding-session query - when the repository is missing; the Codex types cover deliberate and - PreCompact checkpoints, while `session` covers core projections + when the repository is missing; these are agent-authored checkpoints, while + lifecycle envelopes remain local operational trace - active `type=task`, `status=active` - open `type=decision`, `status=open` @@ -73,10 +74,11 @@ Basic Memory for Codex - Redact paths: - Shared hook inbox: - Shared pending envelopes: -- Shared processed envelopes: +- Shared archived envelopes: +- Pending checkpoint requests: - Last flush: - Hook runtime: basic-memory ; uv -- Recent checkpoints: +- Recent checkpoints: - Active tasks: - Open decisions: - Hooks: installed; trust review required in Codex diff --git a/scripts/validate_claude_plugin.py b/scripts/validate_claude_plugin.py index a9a7ea99a..b25d93650 100644 --- a/scripts/validate_claude_plugin.py +++ b/scripts/validate_claude_plugin.py @@ -50,7 +50,8 @@ "bm-status": ( "hook status --harness claude", "pending envelopes", - "processed envelopes", + "archived envelopes", + "Pending checkpoint requests", "last flush", '"type": "session"', '"type": "coding_session"', @@ -195,9 +196,9 @@ def validate_claude_plugin(plugin_dir: Path) -> None: readme = (plugin_dir / "README.md").read_text(encoding="utf-8") for required_text in ( - "normalized `session`", - "`tool_ledger`", - "owned and tested by Basic Memory core", + "lifecycle trace never becomes a graph note", + "archive locally", + "never creates session or tool-ledger notes", ): if required_text not in readme: raise SystemExit(f"README.md: missing schema ownership text {required_text!r}") diff --git a/scripts/validate_codex_plugin.py b/scripts/validate_codex_plugin.py index d8bc55803..5324bec16 100755 --- a/scripts/validate_codex_plugin.py +++ b/scripts/validate_codex_plugin.py @@ -39,21 +39,24 @@ "~/.codex/basic-memory.json", "hook status --harness codex", "pending envelopes", - "processed envelopes", + "archived envelopes", + "Pending checkpoint requests", "last flush", "type=codex_session", "type=coding_session", - "type=session", ), "bm-orient": ( - "Always query `codex_session` and `session`", + "Always query `codex_session`", "type=codex_session", "type=coding_session", - "type=session", ), "bm-checkpoint": ( "Apply the `bm-writing` skill", "A checkpoint is a durable handoff, not a status dump", + "## Privacy Gate", + "`redactKeys` and `redactPaths` accumulate", + "[REDACTED_PATH]", + "skip the checkpoint", "username: ", "hostname: ", "type: coding_session", @@ -73,12 +76,13 @@ ), } REQUIRED_SCHEMAS = ("codex-session.md", "coding-session.md", "decision.md", "task.md") -REQUIRED_HOOK_EVENTS = ("SessionStart", "PreCompact") +REQUIRED_HOOK_EVENTS = ("SessionStart", "PreCompact", "Stop") # Zero-logic shims: the only hook code the plugin ships. The Python bodies # moved into the basic-memory package behind `bm hook` (SPEC-55). REQUIRED_HOOK_SCRIPTS = ( "hooks/session_start.py", "hooks/pre_compact.py", + "hooks/stop.py", ) REQUIRED_SKILL_AGENT_FILES = ("agents/openai.yaml", "assets/icon.svg") REQUIRED_INTERFACE_ASSETS = { @@ -198,7 +202,11 @@ def validate_plugin(plugin_dir: Path) -> None: raise SystemExit(f"{schema_file}: missing entity") readme = (plugin_dir / "README.md").read_text(encoding="utf-8") - for required_text in ("normalized `session`", "`tool_ledger`", "core-owned contracts"): + for required_text in ( + "lifecycle trace stays local", + "Stop hook", + "agent-authored checkpoint", + ): if required_text not in readme: raise SystemExit(f"README.md: missing schema ownership text {required_text!r}") diff --git a/src/basic_memory/cli/commands/hook.py b/src/basic_memory/cli/commands/hook.py index d6ab13e9b..ddaf6e2c0 100644 --- a/src/basic_memory/cli/commands/hook.py +++ b/src/basic_memory/cli/commands/hook.py @@ -3,11 +3,11 @@ Harness plugins reduce to manifests plus one-line shims that exec ``bm hook --harness claude|codex`` with the hook JSON on stdin. All logic lives here: per-harness stdin adapters, the session-start context brief, -the pre-compact checkpoint note, lifecycle-event capture into the inbox WAL, -and the flush/status operator surface. +checkpoint coordination, lifecycle-event capture into the inbox WAL, and the +flush/status operator surface. Contracts: - - Harness verbs (session-start, pre-compact) are fail-open: any error logs + - Harness verbs (session-start, pre-compact, stop) are fail-open: any error logs to stderr and exits 0 — a hook must never disrupt an agent session. - Codex event capture defaults on. An explicit JSON boolean ``false`` turns it off, while malformed values and malformed config fail closed. @@ -73,6 +73,14 @@ class Harness(str, Enum): MAX_SHARED = 6 CODING_SESSION_PROFILE = "coding" CODEX_DEFAULT_CAPTURE_EVENTS = True +CODEX_CHECKPOINT_REASON = ( + "Basic Memory checkpoint required after compaction. Use the " + "`codex:bm-checkpoint` skill now to write one deliberate, durable handoff " + "for the work completed in this turn. Capture the problem, approach, actual " + "changes, verification, decisions, blockers, and next action from the " + "compacted context. Do not write lifecycle telemetry or a transcript dump. " + "Complete the checkpoint before ending the turn." +) @dataclass(frozen=True) @@ -82,9 +90,7 @@ class HarnessProfile: default_recall_timeframe: str default_capture_folder: str session_note_type: str # type stamped on this harness's checkpoint notes - # Types the session-start brief recalls. Includes the generic ``session`` - # the `bm hook flush` projector writes, so flushed/legacy sessions surface - # even when the harness stamps its checkpoints with a distinct type. + # Types the session-start brief recalls from durable, authored checkpoints. recall_session_types: tuple[str, ...] session_id_key: str checkpoint_title_prefix: str @@ -93,7 +99,6 @@ class HarnessProfile: status_hint: str pin_tip: str default_recall_prompt: str - include_workspace_sections: bool # codex adds git status + assistant cursor coding_session_note_type: str @@ -123,16 +128,13 @@ class HarnessProfile: "user makes a material decision, capture it as a note with type: decision. " "Cite permalinks when referencing prior work." ), - include_workspace_sections=False, coding_session_note_type="coding_session", ), Harness.codex: HarnessProfile( default_recall_timeframe="7d", default_capture_folder="codex", session_note_type="codex_session", - # Codex stamps checkpoints codex_session, but the projector writes plain - # `session` — recall both so flushed sessions aren't invisible to Codex. - recall_session_types=("codex_session", "session"), + recall_session_types=("codex_session",), session_id_key="codex_session_id", checkpoint_title_prefix="Codex session", checkpoint_tags=("codex", "auto-capture"), @@ -152,7 +154,6 @@ class HarnessProfile: "Use Basic Memory as durable context, but keep required repo rules in " "AGENTS.md or checked-in docs." ), - include_workspace_sections=True, coding_session_note_type="coding_session", ), } @@ -427,7 +428,7 @@ def _capture_envelope( def _project_query_kwargs(project_ref: str) -> dict[str, str]: - from basic_memory.hooks.projector import split_project_ref + from basic_memory.hooks.project_ref import split_project_ref project, project_id = split_project_ref(project_ref) return {"project_id": project_id} if project_id else {"project": project or project_ref} @@ -481,9 +482,8 @@ async def _gather_context( after_date=timeframe, ) ) - # General and core-projected sessions remain a lower-priority compatibility - # path. They predate required repository metadata and cannot be safely - # narrowed, so coding_session results are always merged first. + # General checkpoints are a lower-priority path because coding_session + # results carry repository identity and are therefore merged first. session_queries.append( _query(project, note_types=list(profile.recall_session_types), after_date=timeframe) ) @@ -532,7 +532,7 @@ def _label(result: dict) -> str: def _readable(ref: str) -> str: - from basic_memory.hooks.projector import UUID_RE + from basic_memory.hooks.project_ref import UUID_RE # Qualified names ("my-team-2/notes") read fine as-is; UUIDs get shortened. return f"shared project {ref[:8]}…" if UUID_RE.match(ref) else ref @@ -710,7 +710,11 @@ def _text_of(content: Any) -> str: if isinstance(content, list): parts = [] for block in content: - if isinstance(block, dict) and block.get("type") == "text": + if isinstance(block, dict) and block.get("type") in { + "text", + "input_text", + "output_text", + }: text = block.get("text") if isinstance(text, str): parts.append(text) @@ -718,12 +722,14 @@ def _text_of(content: Any) -> str: return "" -def _transcript_turns(path: str) -> list[tuple[str, str]]: +def _transcript_turns(path: str, harness: Harness = Harness.claude) -> list[tuple[str, str]]: """Extract (role, text) turns from a JSONL transcript. Skips injected/meta frames and tool results — only real human input and - assistant prose count. Claude Code marks tool results with a - ``toolUseResult`` field and injected/meta turns with ``isMeta``. + assistant prose count. Claude Code stores a top-level ``message``; Codex + stores messages under ``response_item.payload`` with ``input_text`` and + ``output_text`` content blocks. Codex transcripts are not a stable public + API, so this host-specific branch is intentionally narrow and fixture-backed. """ if not path: return [] @@ -738,9 +744,19 @@ def _transcript_turns(path: str) -> list[tuple[str, str]]: obj = json.loads(line) except json.JSONDecodeError: continue - if obj.get("isMeta") or obj.get("toolUseResult") is not None: - continue - msg = obj.get("message") if isinstance(obj.get("message"), dict) else obj + if harness is Harness.codex: + payload = obj.get("payload") + if ( + obj.get("type") != "response_item" + or not isinstance(payload, dict) + or payload.get("type") != "message" + ): + continue + msg = payload + else: + if obj.get("isMeta") or obj.get("toolUseResult") is not None: + continue + msg = obj.get("message") if isinstance(obj.get("message"), dict) else obj role = msg.get("role") or obj.get("type") if role not in ("user", "assistant"): continue @@ -757,23 +773,6 @@ def _clip(value: str, limit: int) -> str: return compact if len(compact) <= limit else compact[: limit - 1].rstrip() + "…" -def _git_status(directory: str) -> list[str]: - """Best-effort working-tree snapshot for Codex checkpoints (read-only).""" - try: - out = subprocess.run( - ["git", "status", "--short"], - cwd=directory or None, - capture_output=True, - text=True, - timeout=5, - ) - except (OSError, subprocess.SubprocessError): - return [] - if out.returncode != 0: - return [] - return [line for line in out.stdout.splitlines() if line.strip()][:20] - - @dataclass(frozen=True, slots=True) class PullRequestContext: number: int @@ -887,17 +886,13 @@ def _checkpoint_note( # (#997: redact obvious secrets before writing artifacts). Redact once at # extraction — every downstream use draws from the redacted strings. # Deferred import: redaction pulls detect-secrets, too heavy for CLI start (#886). - from basic_memory.hooks.redaction import REDACTED_PATH, Redactor + from basic_memory.hooks.redaction import Redactor - # One ruleset for the whole checkpoint: every turn, the cwd, and each git - # status row share the same deny rules, so compile the patterns once here - # rather than per redacted string. + # One ruleset for the whole checkpoint: every turn and the cwd share the + # same deny rules, so compile the patterns once rather than per string. redactor = Redactor.build(extra_redact_paths=extra_redact_paths) user_messages = [redactor.redact_text(text) for role, text in conversation if role == "user"] - assistant_messages = [ - redactor.redact_text(text) for role, text in conversation if role == "assistant" - ] # cwd is a user path too: a session under a configured redactPaths (or a # default deny dir) must not leak the raw path into the note frontmatter or # body. Redact once so both draw from the scrubbed string. @@ -992,23 +987,6 @@ def _checkpoint_note( f"- Pull request: #{coding_context.pull_request.number} — " f"{metadata['pull_request_url']}" ) - if profile.include_workspace_sections: - recent_assistant = assistant_messages[-2:] - if recent_assistant: - body += ["", "## Recent assistant notes"] - body += [f"- {_clip(message, 240)}" for message in recent_assistant] - # Skip the working tree entirely when the workspace itself is denied - # (safe_cwd redacted to the marker means cwd matched a redactPaths entry). - # `git status --short` emits repo-relative filenames with no absolute - # prefix (`M customer-roadmap.md`), so per-row redaction can't match the - # deny path — the whole denied workspace's file list would leak. - status_lines = [] if safe_cwd == REDACTED_PATH else _git_status(working_directory) - if status_lines: - # git status rows carry filenames/paths too — pass them through the - # same floor as the transcript text and cwd (a secret token in a - # filename, or a denied path elsewhere, must not leak into the note). - body += ["", "## Working tree"] - body += [f"- `{redactor.redact_text(line)}`" for line in status_lines] body += [ "", "## Observations", @@ -1074,7 +1052,19 @@ def _pre_compact(harness: Harness, project_dir: Optional[Path]) -> None: if not primary: return - conversation = _transcript_turns(event.transcript_path) + if harness is Harness.codex: + # PreCompact stdout is ignored by Codex, so this hook cannot ask the + # active model to author memory directly. Persist a private request; the + # Stop hook turns it into a one-time continuation prompt after Codex has + # compacted the same turn and still has its summarized working context. + if not event.session_id: + return + from basic_memory.hooks import checkpoint_requests + + checkpoint_requests.create(event.session_id, event.turn_id) + return + + conversation = _transcript_turns(event.transcript_path, harness) # Trigger: nothing usable in the transcript, or no real human turn in it. # Why: an empty or human-less checkpoint is worse than none. Outcome: no-op. if not conversation or not any(role == "user" for role, _ in conversation): @@ -1096,7 +1086,7 @@ def _pre_compact(harness: Harness, project_dir: Optional[Path]) -> None: ) # Deferred import (#886); same internal write path as `bm tool write-note`. - from basic_memory.hooks.projector import split_project_ref + from basic_memory.hooks.project_ref import split_project_ref from basic_memory.mcp.tools import write_note project, project_id = split_project_ref(primary) @@ -1120,6 +1110,55 @@ def _pre_compact(harness: Harness, project_dir: Optional[Path]) -> None: print(f"bm hook pre-compact: checkpoint write failed: {result['error']}", file=sys.stderr) +def _codex_stop_response(payload: dict[str, Any]) -> dict[str, Any]: + """Resolve one Codex Stop event against its pending checkpoint request.""" + session_id = str(payload.get("session_id") or "") + if not session_id: + return {"continue": True} + + from basic_memory.hooks import checkpoint_requests + + request = checkpoint_requests.read(session_id) + if request is None: + return {"continue": True} + + current_turn_id = payload.get("turn_id") + # Trigger: the saved request belongs to an earlier Codex turn, or the + # current Stop cannot prove it belongs to the requesting turn. + # Why: an interrupted continuation can leave the request behind; blocking + # a later turn would author a checkpoint from the wrong working context. + # Outcome: abandon the stale handshake and let the current turn end. + if request.source_turn_id is not None and current_turn_id != request.source_turn_id: + checkpoint_requests.clear(session_id) + return {"continue": True} + + # Trigger: Codex says this Stop is already a continuation. Why: blocking + # again would create a Stop-hook loop. Outcome: clear the local handshake + # and allow the turn to end. Until Codex confirms that continuation state, + # leave the request pending so a failed hook response cannot lose it. + if payload.get("stop_hook_active") is True: + checkpoint_requests.clear(session_id) + return {"continue": True} + + return {"decision": "block", "reason": CODEX_CHECKPOINT_REASON} + + +def _stop(harness: Harness) -> None: + payload = _read_stdin_payload() + response = _codex_stop_response(payload) if harness is Harness.codex else {"continue": True} + print(json.dumps(response, separators=(",", ":"))) + + +def _run_json_fail_open(verb: str, run: Callable[[], None]) -> None: + """Run a JSON-output hook and always leave Codex with a valid response.""" + try: + run() + except (Exception, SystemExit) as exc: + logger.exception(f"bm hook {verb} failed") + print(f"bm hook {verb}: {exc}", file=sys.stderr) + print('{"continue":true}') + + # --- Typer verbs --- HARNESS_OPTION = typer.Option(Harness.claude, "--harness", help="Which harness fired the hook") @@ -1144,10 +1183,16 @@ def pre_compact( harness: Harness = HARNESS_OPTION, project_dir: Optional[Path] = PROJECT_DIR_OPTION, ) -> None: - """Checkpoint the session before compaction; capture an envelope when enabled.""" + """Capture compaction trace and coordinate a durable checkpoint.""" _run_fail_open("pre-compact", lambda: _pre_compact(harness, project_dir)) +@hook_app.command("stop") +def stop(harness: Harness = HARNESS_OPTION) -> None: + """Request one agent-authored Codex checkpoint after compaction.""" + _run_json_fail_open("stop", lambda: _stop(harness)) + + @hook_app.command("flush") def flush( older_than_days: int = typer.Option( @@ -1159,21 +1204,19 @@ def flush( help="Retention window in days for processed and unresolved-pending envelopes", ), ) -> None: - """Project pending inbox envelopes into knowledge-graph artifacts.""" - # Deferred: the projector pulls the envelope stack (detect-secrets) (#886). - from basic_memory.hooks.projector import flush as run_flush + """Archive pending lifecycle envelopes locally; never write graph notes.""" + # Deferred: the archive sweep pulls the envelope stack (detect-secrets) (#886). + from basic_memory.hooks.archive import flush as run_flush result = run_with_cleanup(run_flush(older_than_days=older_than_days)) if result.skipped: typer.echo("flush skipped: another flush is already running") return typer.echo( - f"swept {result.swept} envelope(s): {result.projected} projected, " + f"swept {result.swept} envelope(s): {result.archived} archived, " f"{result.duplicates} duplicate(s), {result.pending} pending, " f"{result.invalid} invalid, {result.pruned} pruned" ) - for note in result.notes: - typer.echo(f" wrote: {note}") # --- install / remove (standalone users, no plugin marketplace) --- @@ -1185,7 +1228,7 @@ def flush( # matches every launcher form we may write — ``basic-memory``, ``bm``, and the # ``uvx "basic-memory>=X"`` fallback — while staying distinctive to our CLI. OWNED_HOOK_COMMAND_RE = re.compile( - r"\bhook\s+(?:session-start|pre-compact)\s+--harness\s+(?:claude|codex)\b" + r"\bhook\s+(?:session-start|pre-compact|stop)\s+--harness\s+(?:claude|codex)\b" ) @@ -1272,6 +1315,7 @@ def group(verb: str, timeout: int, matcher: str | None) -> dict[str, Any]: return { "SessionStart": group("session-start", 30, "startup|resume|compact"), "PreCompact": group("pre-compact", 60, "manual|auto"), + "Stop": group("stop", 30, None), } @@ -1454,7 +1498,7 @@ def status( ) -> None: """Show inbox depth, last flush, settings summary, and tool versions.""" import basic_memory - from basic_memory.hooks import inbox + from basic_memory.hooks import checkpoint_requests, inbox pending = len(inbox.list_envelopes()) processed = len(list(inbox.processed_dir().glob("*.json"))) @@ -1464,7 +1508,8 @@ def status( typer.echo(f"inbox: {inbox.inbox_dir()}") typer.echo(f"pending envelopes: {pending}") - typer.echo(f"processed envelopes: {processed}") + typer.echo(f"archived envelopes: {processed}") + typer.echo(f"pending checkpoint requests: {checkpoint_requests.pending_count()}") typer.echo(f"last flush: {inbox.last_flush() or 'never'}") typer.echo( f"settings ({harness.value}, {mapping_dir}): {'found' if configured else 'not found'}" diff --git a/src/basic_memory/hooks/__init__.py b/src/basic_memory/hooks/__init__.py index ab97da55d..f6090fdf2 100644 --- a/src/basic_memory/hooks/__init__.py +++ b/src/basic_memory/hooks/__init__.py @@ -3,8 +3,8 @@ Agent harnesses (Claude Code, Codex) fire lifecycle hooks; this package is the producer side of the harness WAL. Capture is dumb: hook stdin is normalized by a per-harness adapter, wrapped in a redacted producer envelope, and appended to -the local inbox. Meaning is derived out of band by the deterministic projector -(``bm hook flush``) until the SPEC-54 daemon worker supersedes it. +the local inbox. ``bm hook flush`` retires valid trace into a local audit archive; +durable knowledge is written separately by an active agent or explicit workflow. Modules: - ``_uuid7`` time-ordered event ids (inbox filenames sort chronologically) @@ -12,5 +12,8 @@ - ``redaction`` Stage-1 deterministic redaction floor (always on) - ``inbox`` append-only WAL under the Basic Memory home dir - ``adapters`` per-harness hook stdin normalization - - ``projector`` idempotent inbox -> knowledge-graph projection + - ``archive`` idempotent local audit-archive sweep + - ``projector`` compatibility imports for the retired graph projector + - ``project_ref`` project-name / project-id routing helpers + - ``checkpoint_requests`` private PreCompact -> Stop handoff state for Codex """ diff --git a/src/basic_memory/hooks/archive.py b/src/basic_memory/hooks/archive.py new file mode 100644 index 000000000..3244ca13c --- /dev/null +++ b/src/basic_memory/hooks/archive.py @@ -0,0 +1,97 @@ +"""Retire harness lifecycle envelopes into the local audit archive. + +``bm hook flush`` is an operational WAL sweep, not a memory-authoring path. +Valid envelopes move to the owner-private ``processed/`` archive, while invalid +envelopes remain in the inbox for inspection. Nothing here writes to the graph. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass + +from loguru import logger + +from basic_memory.hooks import inbox +from basic_memory.hooks.envelope import Envelope, envelope_from_json + + +@dataclass +class FlushResult: + """What one local trace sweep did.""" + + swept: int = 0 + archived: int = 0 + duplicates: int = 0 + pending: int = 0 + invalid: int = 0 + pruned: int = 0 + skipped: bool = False + + +def _processed_idempotency_keys() -> set[str]: + """Read replay keys already present in the local audit archive.""" + keys: set[str] = set() + for path in inbox.processed_dir().glob("*.json"): + try: + envelope = envelope_from_json(path.read_text(encoding="utf-8")) + except (OSError, ValueError, json.JSONDecodeError): + continue + keys.add(envelope.idempotency_key) + return keys + + +async def flush(older_than_days: int = inbox.DEFAULT_RETENTION_DAYS) -> FlushResult: + """Archive the pending lifecycle trace without creating graph notes. + + A non-blocking lock serializes sweeps. Valid envelopes are retired locally + regardless of project mapping because no graph routing occurs. Corrupt or + future-versioned envelopes stay pending so an operator can inspect them. + """ + with inbox.flush_lock() as acquired: + if not acquired: + logger.debug("flush skipped: another flush holds the inbox lock") + return FlushResult(skipped=True) + return _flush_locked(older_than_days) + + +def _flush_locked(older_than_days: int) -> FlushResult: + """Archive one inbox snapshot while :func:`flush` holds the lock.""" + result = FlushResult() + seen_keys = _processed_idempotency_keys() + + for path in inbox.list_envelopes(): + result.swept += 1 + try: + text = path.read_text(encoding="utf-8") + except OSError as exc: + # A file can vanish after listing when another process owns it. It + # is neither corrupt nor proof of a failed archive, so skip it. + logger.debug(f"skipping unreadable envelope {path.name}: {exc}") + continue + + try: + envelope: Envelope = envelope_from_json(text) + except (ValueError, json.JSONDecodeError) as exc: + # Unknown trace must remain observable; never delete or reinterpret it. + logger.warning(f"skipping invalid envelope {path.name}: {exc}") + result.invalid += 1 + continue + + duplicate = envelope.idempotency_key in seen_keys + try: + inbox.mark_processed(path) + except OSError as exc: + logger.warning(f"flush could not archive {path.name}: {exc}") + result.pending += 1 + continue + + if duplicate: + result.duplicates += 1 + else: + seen_keys.add(envelope.idempotency_key) + result.archived += 1 + + result.pruned = inbox.prune_processed(older_than_days) + inbox.record_flush() + return result diff --git a/src/basic_memory/hooks/checkpoint_requests.py b/src/basic_memory/hooks/checkpoint_requests.py new file mode 100644 index 000000000..9f2ebcb97 --- /dev/null +++ b/src/basic_memory/hooks/checkpoint_requests.py @@ -0,0 +1,108 @@ +"""Private handoff state between Codex PreCompact and Stop hooks. + +PreCompact cannot ask the active model to author durable memory, and its stdout +is ignored. It records a small local request instead. The next Stop hook turns +that request into a one-time continuation prompt for the same Codex turn. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import uuid +from dataclasses import asdict, dataclass +from datetime import datetime, timezone +from pathlib import Path + +from basic_memory.config import CONFIG_DIR_MODE, CONFIG_FILE_MODE, resolve_data_dir + +REQUESTS_DIR_NAME = "checkpoint-requests" + + +@dataclass(frozen=True, slots=True) +class CheckpointRequest: + source_session_id: str + source_turn_id: str | None + requested_at: str + + +def requests_dir() -> Path: + return resolve_data_dir() / REQUESTS_DIR_NAME + + +def pending_count() -> int: + """Count pending requests for the hook status surface.""" + return sum(1 for path in requests_dir().glob("*.json") if path.is_file()) + + +def _ensure_private_dir() -> Path: + directory = requests_dir() + directory.mkdir(parents=True, exist_ok=True) + if os.name != "nt": + resolve_data_dir().chmod(CONFIG_DIR_MODE) + directory.chmod(CONFIG_DIR_MODE) + return directory + + +def _request_path(session_id: str) -> Path: + if not session_id: + raise ValueError("checkpoint request requires a Codex session id") + digest = hashlib.sha256(f"codex\0{session_id}".encode()).hexdigest() + return requests_dir() / f"{digest}.json" + + +def _write(request: CheckpointRequest) -> Path: + directory = _ensure_private_dir() + target = _request_path(request.source_session_id) + tmp = directory / f"{target.name}.{uuid.uuid4().hex}.tmp" + tmp.write_text(json.dumps(asdict(request), sort_keys=True) + "\n", encoding="utf-8") + if os.name != "nt": + tmp.chmod(CONFIG_FILE_MODE) + os.replace(tmp, target) + return target + + +def create(session_id: str, turn_id: str | None) -> CheckpointRequest: + """Create or replace the pending request for one Codex session.""" + request = CheckpointRequest( + source_session_id=session_id, + source_turn_id=turn_id, + requested_at=datetime.now(timezone.utc).isoformat(timespec="seconds"), + ) + _write(request) + return request + + +def read(session_id: str) -> CheckpointRequest | None: + """Read a pending request, returning ``None`` when none exists.""" + path = _request_path(session_id) + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except FileNotFoundError: + return None + if not isinstance(payload, dict): + raise ValueError(f"invalid checkpoint request: {path}") + try: + source_session_id = payload["source_session_id"] + source_turn_id = payload.get("source_turn_id") + requested_at = payload["requested_at"] + except KeyError as exc: + raise ValueError(f"invalid checkpoint request: {path}") from exc + if ( + not isinstance(source_session_id, str) + or source_session_id != session_id + or (source_turn_id is not None and not isinstance(source_turn_id, str)) + or not isinstance(requested_at, str) + ): + raise ValueError(f"invalid checkpoint request: {path}") + return CheckpointRequest( + source_session_id=source_session_id, + source_turn_id=source_turn_id, + requested_at=requested_at, + ) + + +def clear(session_id: str) -> None: + """Clear a satisfied or deliberately abandoned request.""" + _request_path(session_id).unlink(missing_ok=True) diff --git a/src/basic_memory/hooks/envelope.py b/src/basic_memory/hooks/envelope.py index 4303b6f0d..a06bd16fb 100644 --- a/src/basic_memory/hooks/envelope.py +++ b/src/basic_memory/hooks/envelope.py @@ -1,20 +1,18 @@ """SPEC-55 producer envelope for harness lifecycle events. Adapted from ``plugins/shared/harness_envelope.py`` on the #1064 salvage branch -(credit: sourrrish) — the contract shape, idempotency keying, and provenance -projections proven there carry into core, extended with the 2026-07-15 -revision fields: ``id`` (UUIDv7), ``actor``, ``caused_by``, and -``promotion_status``. - -Envelopes are trace, not memory: they stay ``promotion_status: raw`` until a -projector promotes them. The idempotency key is computed from metadata only, -so redaction never changes identity. +(credit: sourrrish) — the contract shape and idempotency keying proven there +carry into core, extended with the 2026-07-15 revision fields: ``id`` (UUIDv7), +``actor``, ``caused_by``, and ``promotion_status``. + +Envelopes are trace, not memory: they remain ``promotion_status: raw`` and are +archived locally rather than promoted into the graph. The idempotency key is +computed from metadata only, so redaction never changes identity. """ from __future__ import annotations import hashlib -import re from datetime import datetime, timezone from pydantic import BaseModel, ConfigDict, Field, field_validator @@ -25,8 +23,7 @@ ENVELOPE_VERSION = 1 # A harness session, identified by its producing surface and opaque session id. -# The inbox groups, prunes, and routes by this pair, so it earns a name rather -# than an anonymous ``tuple[str, str]`` threaded through the projector. +# Retained as a named compatibility type for trace consumers. type SessionKey = tuple[str, str] # --- Event registry --- @@ -50,9 +47,8 @@ class Envelope(BaseModel): """Normalized event record from a harness lifecycle hook (SPEC-55 Contract 1). - Each field is chosen so the downstream consumer (the projector today, the - SPEC-54 worker later) can coalesce SessionNote / ToolLedger artifacts - without understanding raw hook payload formats. + Each field records stable operational identity without requiring a consumer + to understand raw hook payload formats. This is a persistence boundary: the inbox is a durable WAL that outlives code versions, so parsing must fail fast on junk — a shape mismatch means @@ -71,7 +67,7 @@ class Envelope(BaseModel): source_session_id: str # opaque, surface-defined ts: str # ISO 8601 cwd: str - project_hint: str # consumers fail fast when this doesn't resolve + project_hint: str # mapping at capture time; trace only, never an implicit write route idempotency_key: str # sha256(source:session:event:ts-to-minute)[:16] envelope_version: int = ENVELOPE_VERSION source_turn_id: str | None = None @@ -84,7 +80,7 @@ class Envelope(BaseModel): @classmethod def _supported_version(cls, version: int) -> int: # A future version is a forward-compat signal, not corruption — surface - # it as an error the projector counts and `bm hook status` shows. + # it as an error the archive sweep counts and `bm hook status` shows. if version != ENVELOPE_VERSION: raise ValueError(f"unsupported envelope_version {version!r}") return version @@ -134,7 +130,7 @@ def create_envelope( pass through the Stage-1 redaction floor here, at the factory — no envelope built through this path can carry unredacted payload values or a denied workspace path into the inbox. ``project_hint`` is a project name, not a - path, so it is left intact (the projector resolves against it). + path, so it is left intact as capture-time diagnostic context. """ if event not in V0_EVENTS: raise ValueError(f"Unknown event {event!r}; v0 supports: {sorted(V0_EVENTS)}") @@ -166,57 +162,6 @@ def create_envelope( ) -# --- Projections into Basic Memory artifacts --- - - -def single_line(value: str) -> str: - """Collapse control characters so a value renders as one markdown line. - - Identity fields are opaque, surface-defined strings that skip the payload - redaction floor. A hostile or corrupt value in a fresh capture or a - replayed inbox file must not smuggle extra observation/relation lines into - a projected note body — the same injection the frontmatter path already - blocks by serializing via ``metadata=``. Every dynamic line the projector - emits into note markdown passes through here. - """ - return re.sub(r"[\x00-\x1f\x7f]+", " ", value) - - -def to_provenance_observations(envelope: Envelope) -> list[str]: - """Observation lines stamping an artifact with its producer provenance. - - Appended to a note's "## Observations" section so downstream consumers - (recall, consolidation, memory routines) can trace where the artifact came - from without storing the raw event. The ``[source]`` observation is the one - SPEC-55 requires on every projected artifact. - """ - lines = [ - f"- [source] {single_line(envelope.source)}/{single_line(envelope.source_session_id)}", - f"- [event] {single_line(envelope.event)} at {single_line(envelope.ts)}", - f"- [idempotency] {single_line(envelope.idempotency_key)}", - ] - if envelope.source_turn_id: - lines.append(f"- [turn] {single_line(envelope.source_turn_id)}") - return lines - - -def to_frontmatter_fields(envelope: Envelope) -> dict[str, str]: - """Envelope fields suitable for note frontmatter. - - Makes projected artifacts queryable by source, event, envelope id, and - idempotency key through metadata search. - """ - fields_out = { - "envelope_id": envelope.id, - "envelope_source": envelope.source, - "envelope_event": envelope.event, - "idempotency_key": envelope.idempotency_key, - } - if envelope.source_turn_id: - fields_out["envelope_turn_id"] = envelope.source_turn_id - return fields_out - - # --- Serialization --- @@ -230,7 +175,7 @@ def envelope_from_json(text: str) -> Envelope: Delegates to the model's strict validation (see :class:`Envelope`): a shape mismatch, wrong scalar type, unknown key, or unsupported version raises a - ``pydantic.ValidationError`` — itself a ``ValueError`` — which the projector - and inbox already catch and count. + ``pydantic.ValidationError`` — itself a ``ValueError`` — which the archive + sweep catches and counts. """ return Envelope.model_validate_json(text) diff --git a/src/basic_memory/hooks/inbox.py b/src/basic_memory/hooks/inbox.py index c89f449cb..24b68b0ab 100644 --- a/src/basic_memory/hooks/inbox.py +++ b/src/basic_memory/hooks/inbox.py @@ -26,8 +26,6 @@ from basic_memory.hooks._uuid7 import uuid7_unix_ms from basic_memory.hooks.envelope import ( Envelope, - SessionKey, - envelope_from_json, envelope_to_json, ) @@ -98,7 +96,7 @@ def list_envelopes() -> list[Path]: def mark_processed(path: Path) -> Path: - """Retire a projected envelope into processed/ (kept for audit, then pruned). + """Retire an envelope into the local audit archive (kept, then pruned). Tolerant of a concurrent flush that already retired this envelope: a missing source with the destination already present means another sweep moved it @@ -116,33 +114,6 @@ def mark_processed(path: Path) -> Path: return destination -def _unresolvable_pending_gate( - routable_sessions: frozenset[SessionKey], -) -> Callable[[Path], bool]: - """Build the prune gate: a pending envelope is unresolvable only if it can - *never* flush — it parses, carries no project hint, and its session is not - routable through any sibling. - - Kept (not pruned): a parse failure (corrupt/future-versioned — the trace - ``bm hook status`` surfaces for a human); a present hint (mapped, pending - only because a write failed and must self-heal); and — crucially — a - hint-less file whose ``(source, session_id)`` appears in ``routable_sessions`` - (another envelope in that session, pending or already processed, carries a - hint, so the group self-heals and rebuilds the full session on a later sweep). - """ - - def gate(path: Path) -> bool: - try: - envelope = envelope_from_json(path.read_text(encoding="utf-8")) - except (OSError, ValueError): # ValueError covers json.JSONDecodeError - return False - if envelope.project_hint.strip(): - return False - return envelope.session_key not in routable_sessions - - return gate - - def _prune_dir( directory: Path, older_than_days: int, @@ -186,35 +157,6 @@ def prune_processed(older_than_days: int = DEFAULT_RETENTION_DAYS) -> int: return _prune_dir(processed_dir(), older_than_days) -def prune_pending( - older_than_days: int = DEFAULT_RETENTION_DAYS, - routable_sessions: frozenset[SessionKey] = frozenset(), -) -> int: - """Delete pending envelopes older than the retention window. - - A session that never resolves a project mapping (``primaryProject`` unset for - its whole lifetime) produces envelopes the projector can never route — it - holds them pending, waiting for a mapping that, for a fully-unmapped session, - never comes. Bounding the inbox by the same window the processed side already - uses keeps that unresolvable trace from accumulating without limit, while - still giving a mapping the full window to appear (a later same-session - capture carrying a hint resolves the whole group via the projector's merge). - - ``routable_sessions`` is the set of ``(source, session_id)`` the caller knows - to be routable (a hinted envelope somewhere in the session — pending or - processed). Only *unresolvable* pending entries are pruned: a corrupt file, a - mapped write-failure, and a hint-less file belonging to a routable session - are all left in place, so retention never defeats self-heal (the session - still rebuilds in full on a later sweep) or eats the corruption trace - ``bm hook status`` surfaces. - """ - return _prune_dir( - inbox_dir(), - older_than_days, - should_prune=_unresolvable_pending_gate(routable_sessions), - ) - - # --- Flush bookkeeping (the `bm hook status` debuggability surface) --- @@ -239,10 +181,9 @@ def last_flush() -> str | None: def flush_lock() -> Iterator[bool]: """Hold an exclusive advisory lock over the inbox for the duration of a flush. - Two overlapping ``bm hook flush`` runs can interleave their list → write → - retire steps so a stale sweep overwrites a SessionNote/ToolLedger without a - sibling's just-retired event, dropping that event's row until another event - arrives in the session. Serializing flushes closes the race. + Two overlapping ``bm hook flush`` runs can classify the same replay before + either retires it. Serializing the list → classify → archive sweep keeps the + local dedup counts deterministic. Yields ``True`` to the holder and ``False`` to any flush that arrives while the lock is held — that flush skips rather than blocks, because the holder diff --git a/src/basic_memory/hooks/project_ref.py b/src/basic_memory/hooks/project_ref.py new file mode 100644 index 000000000..0d8d597fd --- /dev/null +++ b/src/basic_memory/hooks/project_ref.py @@ -0,0 +1,16 @@ +"""Basic Memory project-reference routing shared by hook workflows.""" + +from __future__ import annotations + +import re + +UUID_RE = re.compile( + r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", re.IGNORECASE +) + + +def split_project_ref(ref: str) -> tuple[str | None, str | None]: + """Split a project reference into the ``(project, project_id)`` routing pair.""" + if UUID_RE.match(ref): + return None, ref + return ref, None diff --git a/src/basic_memory/hooks/projector.py b/src/basic_memory/hooks/projector.py index ca4b0d077..b50da25e0 100644 --- a/src/basic_memory/hooks/projector.py +++ b/src/basic_memory/hooks/projector.py @@ -1,469 +1,11 @@ -"""Deterministic projector: sweep the inbox into knowledge-graph artifacts. +"""Compatibility imports for the retired lifecycle-event projector. -The interim consumer of the harness WAL (``bm hook flush``) until the SPEC-54 -daemon worker lands. No LLM: SessionNote skeletons and ToolLedger entries are -derived mechanically from the captured envelopes. - -Idempotent by construction (the EverOS pattern): envelopes are treated as -hints — dedup on ``idempotency_key``, artifacts re-derived with deterministic -titles and ``overwrite=True`` — so WAL replays and duplicate hooks can never -corrupt or double-write. Every run sweeps the whole inbox, so envelopes -captured while nothing was consuming self-heal; there is no missed-event -window. - -Writes go through the same internal write path the CLI's ``write-note`` uses -(the MCP ``write_note`` tool via the async client) — never a subprocess. +Lifecycle trace is no longer promoted into graph notes. New code should import +the local archive sweep from :mod:`basic_memory.hooks.archive` and project-ref +routing from :mod:`basic_memory.hooks.project_ref`. """ -from __future__ import annotations - -import json -import os -import re -from dataclasses import dataclass, field -from getpass import getuser -from pathlib import Path -from socket import gethostname - -from loguru import logger - -from basic_memory.hooks import inbox -from basic_memory.hooks.envelope import ( - Envelope, - SessionKey, - envelope_from_json, - single_line, - to_frontmatter_fields, - to_provenance_observations, -) - -# Cloud project refs come in two unambiguous forms (names collide across -# workspaces): a workspace-qualified name routes via project, an external_id -# UUID via project_id. Mirrors the routing the hook scripts used. -UUID_RE = re.compile( - r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", re.IGNORECASE -) - -DEFAULT_CAPTURE_FOLDER = "sessions" -CREATED_BY_PREFIX = "bm-hook" - -# Artifact note types. These are the explicit `note_type` write_note persists — -# the `type:` in the rendered frontmatter is stripped and replaced by this arg, -# so both must agree or recall (search by type) can't find projected notes. -SESSION_NOTE_TYPE = "session" -TOOL_LEDGER_NOTE_TYPE = "tool_ledger" - - -@dataclass -class FlushResult: - """What one projector sweep did, for `bm hook flush` / `status` reporting.""" - - swept: int = 0 # envelope files seen in the inbox - projected: int = 0 # envelopes promoted into artifacts - duplicates: int = 0 # idempotency-key replays retired without writing - pending: int = 0 # left in the inbox (no project mapping, or write failed) - invalid: int = 0 # unreadable envelope files left in place - pruned: int = 0 # envelopes removed by retention (processed + unresolved pending) - skipped: bool = False # another flush held the inbox lock; this run did nothing - notes: list[str] = field(default_factory=list) # artifact titles written - - -def split_project_ref(ref: str) -> tuple[str | None, str | None]: - """Split a project reference into the (project, project_id) routing pair. - - A UUID reference must route via ``project_id``, not ``project``, or the - call silently fails to land in a UUID-configured project. - """ - if UUID_RE.match(ref): - return None, ref - return ref, None - - -def _processed_envelopes_by_session() -> dict[SessionKey, list[Envelope]]: - """Already-projected envelopes, grouped by session key. - - A session's artifacts are overwritten in full on every sweep, so re-deriving - them from only the still-pending envelopes would drop everything projected on - an earlier sweep (session_started vanishing once session_ended arrives). - Reloading the processed envelopes lets each sweep rebuild the note from the - complete session history (bounded by retention). Corrupt processed files are - skipped, never deleted. - """ - grouped: dict[SessionKey, list[Envelope]] = {} - for path in inbox.processed_dir().glob("*.json"): - try: - envelope = envelope_from_json(path.read_text(encoding="utf-8")) - except (OSError, ValueError, json.JSONDecodeError): - continue - grouped.setdefault(envelope.session_key, []).append(envelope) - return grouped - - -def _session_label(session_id: str) -> str: - # Full id, not a prefix: two sessions sharing an 8-char prefix would derive - # the same title, the same permalink, and clobber each other's notes. - return session_id or "unknown" - - -def _dedup_by_key(envelopes: list[Envelope]) -> list[Envelope]: - """Keep one envelope per idempotency key, preserving input order. - - A retired replay lives in ``processed/`` next to its original (same key, - distinct id), so a full rebuild that merges processed history would render - both as duplicate rows. Deduping the merged list — earliest first, since the - caller sorts by chronological uuid7 id — keeps the original and drops the - replay, honoring the idempotency contract. - """ - seen: set[str] = set() - unique: list[Envelope] = [] - for envelope in envelopes: - if envelope.idempotency_key in seen: - continue - seen.add(envelope.idempotency_key) - unique.append(envelope) - return unique - - -def _capture_folder(envelopes: list[Envelope]) -> str: - # Capture embeds the harness's configured folder into the payload so the - # projector needs no settings access of its own. - for envelope in envelopes: - folder = envelope.payload.get("capture_folder") - if isinstance(folder, str) and folder.strip(): - return folder.strip() - return DEFAULT_CAPTURE_FOLDER - - -def _artifact_username() -> str: - try: - return getuser() - except (OSError, KeyError): - # Trigger: the runtime has no passwd entry for its uid. Python 3.12 may - # raise KeyError here; Python 3.13+ normalizes the failure to OSError. - # Why: optional provenance must never prevent queued notes from flushing. - # Outcome: use the host-provided identity, or a stable unknown value. - return os.environ.get("USER") or os.environ.get("USERNAME") or "unknown" - - -def _artifact_metadata(first: Envelope) -> dict[str, str]: - """Provenance frontmatter every projected artifact carries. - - Returned as a dict for write_note to serialize (``metadata=``), never - hand-built into a ``---`` block: ``source_turn_id`` is opaque, - surface-defined text (via ``to_frontmatter_fields``), so a value with - YAML-special content — ``turn: 42``, a colon, a newline — would make a - hand-built block invalid and wedge the whole session pending on every flush - retry. The note ``type`` rides the ``note_type`` write_note arg, so it is not - repeated here. - """ - metadata = { - "created_by": f"{CREATED_BY_PREFIX}/{first.source}", - "caused_by_event": first.id, - "username": _artifact_username(), - "hostname": gethostname(), - } - metadata.update(to_frontmatter_fields(first)) - return metadata - - -def _session_note( - source: str, session_id: str, envelopes: list[Envelope] -) -> tuple[str, str, dict[str, str]]: - """Derive the SessionNote skeleton (title, body, metadata) for one group.""" - first = envelopes[0] - # single_line on every dynamic line: session ids and replayed envelope - # fields are opaque text, and an embedded newline would parse as an extra - # observation/relation in the projected note (same class as the - # provenance-observation injection fixed in to_provenance_observations). - title = single_line(f"Session {_session_label(session_id)} ({source})") - metadata = _artifact_metadata(first) - # status/open mirrors the checkpoint notes so structured recall finds both. - metadata["status"] = "open" - - body = [ - "", - f"# {title}", - "", - "_Session skeleton projected from captured harness events by `bm hook flush`._", - "", - "## Events", - *[ - single_line(f"- {envelope.event} at {envelope.ts} (`{envelope.id}`)") - for envelope in envelopes - ], - "", - "## Observations", - *to_provenance_observations(first), - ] - return title, "\n".join(body), metadata - - -def _tool_ledger_note( - source: str, session_id: str, envelopes: list[Envelope] -) -> tuple[str, str, dict[str, str]]: - """Derive the ToolLedger (title, body, metadata) for one session group. - - V0 captures only lifecycle events, so the ledger records those; tool_called - entries join when PostToolUse capture lands. - """ - first = envelopes[0] - # single_line for the same reason as _session_note: no replayed envelope - # field may break out of its markdown line. - title = single_line(f"Tool Ledger {_session_label(session_id)} ({source})") - metadata = _artifact_metadata(first) - - entries = [ - single_line( - f"- [event] {envelope.event} at {envelope.ts} " - f"(actor: {envelope.actor}, idempotency: {envelope.idempotency_key})" - ) - for envelope in envelopes - ] - body = [ - "", - f"# {title}", - "", - "_Event ledger projected from captured harness events by `bm hook flush`._", - "", - "## Entries", - *entries, - "", - "## Observations", - single_line(f"- [source] {source}/{session_id}"), - ] - return title, "\n".join(body), metadata - - -async def _write_artifact( - title: str, - content: str, - folder: str, - project_hint: str, - note_type: str, - metadata: dict[str, str], -) -> None: - # Deferred: importing basic_memory.mcp.tools loads the whole tool stack - # (fastmcp, SQLAlchemy) and must not happen at CLI import time (#886). - from basic_memory.mcp.tools import write_note - - project, project_id = split_project_ref(project_hint) - result = await write_note( - title=title, - content=content, - directory=folder, - project=project, - project_id=project_id, - tags=["auto-capture"], - # Explicit note_type: write_note strips `type:` from the content - # frontmatter and persists this arg instead. Without it the artifact - # lands as the default `note`, invisible to session/ledger type recall. - note_type=note_type, - # Provenance goes through metadata= so write_note serializes YAML-special - # values safely (see _artifact_metadata) instead of a hand-built block. - metadata=metadata, - overwrite=True, - output_format="json", - ) - # write_note reports failures as an error field in JSON mode; surface it as - # an exception so the group stays pending instead of being retired unwritten. - if isinstance(result, dict) and result.get("error"): - raise RuntimeError(f"write_note failed for {title!r}: {result['error']}") - - -async def flush(older_than_days: int = inbox.DEFAULT_RETENTION_DAYS) -> FlushResult: - """Sweep the whole inbox and project it into artifacts. - - Envelopes without a resolvable project mapping stay pending — fail fast, - never write to the wrong project. Groups whose write fails also stay - pending and self-heal on the next sweep. - - Held under an exclusive inbox lock: a flush that arrives while another is - running skips rather than racing it (a stale snapshot could overwrite an - artifact without a sibling's just-retired event). The running flush sweeps - everything, so the skipped run loses no work. - """ - with inbox.flush_lock() as acquired: - if not acquired: - logger.debug("flush skipped: another flush holds the inbox lock") - return FlushResult(skipped=True) - return await _flush_locked(older_than_days) - - -async def _flush_locked(older_than_days: int) -> FlushResult: - """Project the inbox once, under the flush lock held by :func:`flush`.""" - result = FlushResult() - - # --- Load the inbox in capture order --- - entries: list[tuple[Path, Envelope]] = [] - for path in inbox.list_envelopes(): - result.swept += 1 - try: - text = path.read_text(encoding="utf-8") - except OSError as exc: - # Trigger: the file vanished (a concurrent flush retired it) or is - # transiently unreadable between listing and reading. - # Why: one missing/locked file must never abort the whole sweep and - # skip the remaining valid envelopes + the flush marker. - # Outcome: skip it — it isn't corrupt trace, just gone or busy; the - # sweep that owns it handles it. Not counted as invalid. - logger.debug(f"skipping unreadable envelope {path.name}: {exc}") - continue - try: - entries.append((path, envelope_from_json(text))) - except (ValueError, json.JSONDecodeError) as exc: - # Trigger: corrupt or future-versioned envelope file. - # Why: deleting it would destroy trace; projecting it would guess. - # Outcome: left in place, counted, visible in `bm hook status`. - logger.warning(f"skipping invalid envelope {path.name}: {exc}") - result.invalid += 1 - - # --- Group by session, preserving capture order within each group --- - groups: dict[SessionKey, list[tuple[Path, Envelope]]] = {} - for path, envelope in entries: - groups.setdefault(envelope.session_key, []).append((path, envelope)) - - processed_by_session = _processed_envelopes_by_session() - seen_keys = { - envelope.idempotency_key - for envelopes in processed_by_session.values() - for envelope in envelopes - } - - # Sessions with a hint *anywhere* (a pending envelope this sweep, or an - # already-processed one) are routable: a hint-less pending envelope in such a - # session self-heals via the merge, so retention must not prune it even when - # it's past the window (e.g. captured before primaryProject was set, during a - # prolonged write outage). Only fully-unmapped sessions are prunable. - routable_sessions = frozenset( - key - for source_map in ( - {k: [e for _, e in v] for k, v in groups.items()}, - processed_by_session, - ) - for key, envelopes in source_map.items() - if any(envelope.project_hint.strip() for envelope in envelopes) - ) - - for session_key, group in groups.items(): - source, session_id = session_key - # --- Dedup: envelopes are hints, never double-write --- - # Two replay kinds, retired at different times: one duplicating an - # already-projected envelope (durable in processed/ — safe to retire now) - # and one duplicating an in-group sibling being projected this sweep - # (safe only once that sibling's write succeeds). - fresh: list[tuple[Path, Envelope]] = [] - processed_replays: list[Path] = [] - group_replays: list[tuple[Path, Envelope]] = [] - group_keys: set[str] = set() - for path, envelope in group: - if envelope.idempotency_key in seen_keys: - processed_replays.append(path) - elif envelope.idempotency_key in group_keys: - group_replays.append((path, envelope)) - else: - group_keys.add(envelope.idempotency_key) - fresh.append((path, envelope)) - - # A replay of an already-projected envelope is safe to retire immediately. - for path in processed_replays: - inbox.mark_processed(path) - result.duplicates += 1 - - # No fresh work means no in-group siblings to project, so group_replays is - # empty here (a key's first occurrence is always the fresh one). - if not fresh: - continue - - fresh_envelopes = [envelope for _, envelope in fresh] - # Rebuild from the COMPLETE session: previously-projected envelopes (now - # in processed/) merged with the fresh ones in capture order (uuid7 ids - # sort chronologically). Overwriting from fresh alone would erase events - # projected on an earlier sweep. Dedup by idempotency key so a replay - # that was retired into processed/ next to its original doesn't resurface - # as a duplicate row here. - prior = processed_by_session.get(session_key, []) - replay_envelopes = [envelope for _, envelope in group_replays] - # Routing scans the COMPLETE, UN-deduped history — prior processed - # envelopes, fresh, and in-group replays alike. A mapping can land on the - # later same-key capture (primaryProject set between two same-minute - # hooks) while the first is unmapped, and _dedup_by_key keeps the earlier - # (hint-less) one for the rendered rows — so scanning the deduped list - # would miss a hint that only the dropped replay carries, leaving the - # group pending forever (routable_sessions also spares it from pruning). - project_hint = next( - ( - envelope.project_hint.strip() - for envelope in (*prior, *fresh_envelopes, *replay_envelopes) - if envelope.project_hint.strip() - ), - "", - ) - # Rendered rows still dedup by key so a retired replay in processed/ - # doesn't resurface as a duplicate row. - envelopes = _dedup_by_key(sorted(prior + fresh_envelopes, key=lambda envelope: envelope.id)) - if not project_hint: - # Trigger: no project mapping resolved for this session. - # Why: writing to a default/guessed project would put trace in the - # wrong graph — the one unrecoverable failure mode. - # Outcome: envelopes stay pending — a later same-session capture may - # carry a hint (resolving the whole group via the merge above), - # and retention prunes them if none ever does. Retire nothing, - # including group_replays: they must self-heal alongside fresh. - result.pending += len(fresh) - continue - - session_title, session_content, session_metadata = _session_note( - source, session_id, envelopes - ) - ledger_title, ledger_content, ledger_metadata = _tool_ledger_note( - source, session_id, envelopes - ) - folder = _capture_folder(envelopes) - try: - await _write_artifact( - session_title, - session_content, - folder, - project_hint, - SESSION_NOTE_TYPE, - session_metadata, - ) - await _write_artifact( - ledger_title, - ledger_content, - folder, - project_hint, - TOOL_LEDGER_NOTE_TYPE, - ledger_metadata, - ) - except Exception as exc: - # Trigger: the write path failed (project missing, API error, ...). - # Why: retiring unwritten envelopes would silently drop events. - # Outcome: group stays pending; the next sweep re-derives it. Leave - # group_replays pending too — retiring them now would let the - # next sweep read their key from processed/ and wrongly retire - # the still-unwritten fresh envelope as a replay. - logger.warning(f"flush left {source}/{session_id} pending: {exc}") - result.pending += len(fresh) - continue - - # Write succeeded: retire the fresh envelopes and only now the in-group - # replays they duplicated. - for path, _ in fresh: - inbox.mark_processed(path) - result.projected += 1 - for path, _ in group_replays: - inbox.mark_processed(path) - result.duplicates += 1 - result.notes += [session_title, ledger_title] +from basic_memory.hooks.archive import FlushResult, flush +from basic_memory.hooks.project_ref import UUID_RE, split_project_ref - # Retire both sides on the same window: processed audit copies, and pending - # trace from fully-unmapped sessions (so the inbox can't grow without limit). - # routable_sessions are spared — a hint-less file whose session is routable - # self-heals, so pruning it would drop events from a session that still - # rebuilds in full on a later successful sweep. - result.pruned = inbox.prune_processed(older_than_days) + inbox.prune_pending( - older_than_days, routable_sessions - ) - inbox.record_flush() - return result +__all__ = ["FlushResult", "UUID_RE", "flush", "split_project_ref"] diff --git a/tests/cli/test_coding_session_context.py b/tests/cli/test_coding_session_context.py index 633b6a741..d80f263d0 100644 --- a/tests/cli/test_coding_session_context.py +++ b/tests/cli/test_coding_session_context.py @@ -29,18 +29,6 @@ def _git_repo(tmp_path: Path) -> Path: return repository -def _write_config(repo_path: Path, **overrides: object) -> None: - config: dict[str, object] = { - "primaryProject": "demo", - "sessionProfile": "coding", - "repository": "basicmachines-co/basic-memory", - **overrides, - } - (repo_path / ".codex" / "basic-memory.json").write_text( - json.dumps({"basicMemory": config}), encoding="utf-8" - ) - - def _write_claude_config(repo_path: Path, **overrides: object) -> None: config: dict[str, object] = { "primaryProject": "demo", @@ -78,13 +66,8 @@ def _payload(repository: Path, transcript: Path) -> str: ) -def test_coding_profile_writes_required_git_and_pull_request_frontmatter( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setenv("BASIC_MEMORY_CONFIG_DIR", str(tmp_path / "bm-home")) +def test_coding_context_reads_required_git_and_pull_request_metadata(tmp_path: Path) -> None: repository = _git_repo(tmp_path) - _write_config(repository) - transcript = _transcript(tmp_path) pull_request = hook_module.PullRequestContext( number=1124, title="feat(plugins): add coding sessions", @@ -93,21 +76,11 @@ def test_coding_profile_writes_required_git_and_pull_request_frontmatter( base_branch="main", head_branch="feature", ) - mock_write = AsyncMock(return_value={"action": "created"}) - with ( - patch("basic_memory.mcp.tools.write_note", mock_write), - patch.object(hook_module, "_pull_request_context", return_value=pull_request), - ): - result = runner.invoke( - cli_app, - ["hook", "pre-compact", "--harness", "codex", "--project-dir", str(repository)], - input=_payload(repository, transcript), + with patch.object(hook_module, "_pull_request_context", return_value=pull_request): + context = hook_module._coding_context( + {"repository": "basicmachines-co/basic-memory"}, str(repository) ) - assert result.exit_code == 0 - assert mock_write.await_args is not None - kwargs = mock_write.await_args.kwargs - metadata = kwargs["metadata"] expected_sha = subprocess.run( ["git", "rev-parse", "HEAD"], cwd=repository, @@ -115,45 +88,22 @@ def test_coding_profile_writes_required_git_and_pull_request_frontmatter( capture_output=True, text=True, ).stdout.strip() - assert kwargs["note_type"] == "coding_session" - assert metadata["project"] == "demo" - assert metadata["repository"] == "basicmachines-co/basic-memory" - assert metadata["repo_root"] - assert metadata["cwd"] - assert metadata["repo_root"] == metadata["cwd"] - assert metadata["branch"] == "feature" - assert metadata["git_sha"] == expected_sha - assert metadata["pull_request_number"] == "1124" - assert metadata["pull_request_state"] == "open" - assert metadata["pull_request_base"] == "main" - assert metadata["pull_request_head"] == "feature" - assert "## Repository" in kwargs["content"] - assert "Pull request: #1124" in kwargs["content"] - - -def test_coding_profile_omits_pull_request_fields_when_branch_has_no_pr( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setenv("BASIC_MEMORY_CONFIG_DIR", str(tmp_path / "bm-home")) + assert context.repository == "basicmachines-co/basic-memory" + assert context.repo_root == repository.as_posix() + assert context.branch == "feature" + assert context.git_sha == expected_sha + assert context.pull_request == pull_request + + +def test_coding_context_omits_pull_request_when_branch_has_no_pr(tmp_path: Path) -> None: repository = _git_repo(tmp_path) - _write_config(repository) - transcript = _transcript(tmp_path) - mock_write = AsyncMock(return_value={"action": "created"}) - with ( - patch("basic_memory.mcp.tools.write_note", mock_write), - patch.object(hook_module, "_pull_request_context", return_value=None), - ): - result = runner.invoke( - cli_app, - ["hook", "pre-compact", "--harness", "codex", "--project-dir", str(repository)], - input=_payload(repository, transcript), + with patch.object(hook_module, "_pull_request_context", return_value=None): + context = hook_module._coding_context( + {"repository": "basicmachines-co/basic-memory"}, str(repository) ) - assert result.exit_code == 0 - assert mock_write.await_args is not None - metadata = mock_write.await_args.kwargs["metadata"] - assert metadata["repository"] == "basicmachines-co/basic-memory" - assert "pull_request_number" not in metadata + assert context.repository == "basicmachines-co/basic-memory" + assert context.pull_request is None def test_claude_coding_profile_writes_coding_session( @@ -182,24 +132,13 @@ def test_claude_coding_profile_writes_coding_session( assert kwargs["metadata"]["claude_session_id"] == "session-1" -def test_coding_profile_requires_confirmed_repository( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setenv("BASIC_MEMORY_CONFIG_DIR", str(tmp_path / "bm-home")) +def test_coding_context_requires_confirmed_repository(tmp_path: Path) -> None: repository = _git_repo(tmp_path) - _write_config(repository, repository="") - transcript = _transcript(tmp_path) - mock_write = AsyncMock() - with patch("basic_memory.mcp.tools.write_note", mock_write): - result = runner.invoke( - cli_app, - ["hook", "pre-compact", "--harness", "codex", "--project-dir", str(repository)], - input=_payload(repository, transcript), - ) - assert result.exit_code == 0 - assert "coding session profile requires basicMemory.repository" in result.stderr - mock_write.assert_not_awaited() + with pytest.raises( + RuntimeError, match="coding session profile requires basicMemory.repository" + ): + hook_module._coding_context({"repository": ""}, str(repository)) def test_coding_profile_uses_dedicated_schema_for_both_harnesses() -> None: @@ -210,14 +149,14 @@ def test_coding_profile_uses_dedicated_schema_for_both_harnesses() -> None: assert claude.coding_session_note_type == "coding_session" -def test_coding_recall_filters_by_repository_and_merges_legacy_sessions() -> None: +def test_coding_recall_filters_by_repository_and_merges_codex_sessions() -> None: queries: list[dict[str, object]] = [] async def fake_query(project: str | None, **filters: object) -> dict: queries.append({"project": project, **filters}) if filters.get("note_types") == ["coding_session"]: return {"results": [{"title": "Coding", "permalink": "sessions/coding"}]} - if filters.get("note_types") == ["codex_session", "session"]: + if filters.get("note_types") == ["codex_session"]: return { "results": [ {"title": "Duplicate", "permalink": "sessions/coding"}, @@ -241,6 +180,7 @@ async def fake_query(project: str | None, **filters: object) -> dict: coding_query = next(query for query in queries if query.get("note_types") == ["coding_session"]) assert coding_query["metadata_filters"] == {"repository": "basicmachines-co/basic-memory"} assert [row["title"] for row in hook_module._rows(context.sessions)] == ["Coding", "Legacy"] + assert all(query.get("note_types") != ["codex_session", "session"] for query in queries) def test_coding_recall_requires_configured_repository() -> None: diff --git a/tests/cli/test_hook_command.py b/tests/cli/test_hook_command.py index d914e9180..a45945dc8 100644 --- a/tests/cli/test_hook_command.py +++ b/tests/cli/test_hook_command.py @@ -89,6 +89,40 @@ def _transcript(tmp_path: Path) -> Path: return path +def _codex_transcript(tmp_path: Path) -> Path: + """A sanitized fixture matching Codex's current response_item JSONL shape.""" + lines = [ + {"type": "session_meta", "payload": {"id": "s-abc12345"}}, + { + "type": "response_item", + "payload": { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "Fix the login bug"}], + }, + }, + { + "type": "response_item", + "payload": {"type": "reasoning", "summary": ["private reasoning"]}, + }, + { + "type": "response_item", + "payload": { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "Found the null check issue"}], + }, + }, + { + "type": "response_item", + "payload": {"type": "function_call_output", "output": "tool noise"}, + }, + ] + path = tmp_path / "codex-transcript.jsonl" + path.write_text("\n".join(json.dumps(line) for line in lines), encoding="utf-8") + return path + + def _init_git_repo(project: Path) -> None: subprocess.run(["git", "init"], cwd=project, check=True, capture_output=True) @@ -406,28 +440,24 @@ def test_session_start_codex_profile(bm_home: Path, tmp_path: Path) -> None: ) assert result.exit_code == 0 - # Codex recalls its own codex_session checkpoints *and* the generic `session` - # notes the flush projector writes, over a 7d default window. + # Codex recalls durable checkpoints, not locally archived lifecycle trace. session_query = mock_search.await_args_list[2].kwargs - assert session_query["note_types"] == ["codex_session", "session"] + assert session_query["note_types"] == ["codex_session"] assert session_query["after_date"] == "7d" assert "codex/" in result.stdout -def test_session_start_codex_recalls_flushed_generic_session(bm_home: Path, tmp_path: Path) -> None: - # Regression: `bm hook flush` writes session artifacts as type `session`. - # The Codex brief must recall them, not only `codex_session` checkpoints. +def test_session_start_codex_does_not_query_lifecycle_trace(bm_home: Path, tmp_path: Path) -> None: project = tmp_path / "codex-proj" (project / ".codex").mkdir(parents=True) (project / ".codex" / "basic-memory.json").write_text( json.dumps({"basicMemory": {"primaryProject": "demo"}}), encoding="utf-8" ) - flushed = _search_result("Session s-1 (codex)") with patch( "basic_memory.mcp.tools.search_notes", new_callable=AsyncMock, - side_effect=[SEARCH_EMPTY, SEARCH_EMPTY, flushed], - ): + return_value=SEARCH_EMPTY, + ) as mock_search: result = runner.invoke( cli_app, ["hook", "session-start", "--harness", "codex", "--project-dir", str(project)], @@ -435,7 +465,7 @@ def test_session_start_codex_recalls_flushed_generic_session(bm_home: Path, tmp_ ) assert result.exit_code == 0 - assert "Session s-1 (codex)" in result.stdout + assert mock_search.await_args_list[2].kwargs["note_types"] == ["codex_session"] # --- session-start / pre-compact: envelope capture gate --- @@ -701,7 +731,7 @@ def test_pre_compact_missing_transcript_is_silent(bm_home: Path, claude_project: def test_pre_compact_captures_envelope_even_without_mapping(bm_home: Path, tmp_path: Path) -> None: # Capture is dumb: an unmapped session is still trace worth keeping; the - # projector holds it pending until a mapping resolves. + # The trace remains useful even though no durable checkpoint can be routed. project = tmp_path / "unmapped" (project / ".claude").mkdir(parents=True) (project / ".claude" / "settings.json").write_text( @@ -739,7 +769,11 @@ def test_pre_compact_surfaces_write_error_on_stderr( assert "checkpoint write failed" in result.stderr -def test_pre_compact_codex_includes_workspace_sections(bm_home: Path, tmp_path: Path) -> None: +def test_codex_pre_compact_requests_agent_authored_checkpoint_at_stop( + bm_home: Path, tmp_path: Path +) -> None: + from basic_memory.hooks import checkpoint_requests + project = tmp_path / "codex-proj" (project / ".codex").mkdir(parents=True) _init_git_repo(project) @@ -747,12 +781,9 @@ def test_pre_compact_codex_includes_workspace_sections(bm_home: Path, tmp_path: json.dumps({"primaryProject": "demo"}), encoding="utf-8", # flat form, no basicMemory key ) - transcript = _transcript(tmp_path) + transcript = _codex_transcript(tmp_path) mock_write = AsyncMock(return_value={"action": "created"}) - with ( - patch("basic_memory.mcp.tools.write_note", mock_write), - patch.object(hook_module, "_git_status", return_value=["M src/app.py"]), - ): + with patch("basic_memory.mcp.tools.write_note", mock_write): result = runner.invoke( cli_app, ["hook", "pre-compact", "--harness", "codex", "--project-dir", str(project)], @@ -766,85 +797,74 @@ def test_pre_compact_codex_includes_workspace_sections(bm_home: Path, tmp_path: ) assert result.exit_code == 0 - assert mock_write.await_args is not None - kwargs = mock_write.await_args.kwargs - assert kwargs["directory"] == "codex/codex-proj" - assert kwargs["tags"] == ["codex", "auto-capture"] - assert kwargs["title"].startswith("Codex session ") - assert kwargs["note_type"] == "codex_session" - assert kwargs["metadata"]["codex_session_id"] == "s-abc12345" - assert kwargs["metadata"]["codex_turn_id"] == "turn-42" - assert kwargs["metadata"]["model"] == "gpt-5.2-codex" - content = kwargs["content"] - assert "## Recent assistant notes" in content - assert "## Working tree" in content - assert "- `M src/app.py`" in content + mock_write.assert_not_awaited() + request = checkpoint_requests.read("s-abc12345") + assert request is not None + assert request.source_turn_id == "turn-42" + + first_stop = runner.invoke( + cli_app, + ["hook", "stop", "--harness", "codex"], + input=_payload(project, turn_id="turn-42", stop_hook_active=False), + ) + assert first_stop.exit_code == 0 + response = json.loads(first_stop.stdout) + assert response["decision"] == "block" + assert "`codex:bm-checkpoint`" in response["reason"] + assert "lifecycle telemetry" in response["reason"] + assert checkpoint_requests.read("s-abc12345") is not None + + second_stop = runner.invoke( + cli_app, + ["hook", "stop", "--harness", "codex"], + input=_payload(project, turn_id="turn-42", stop_hook_active=True), + ) -def test_pre_compact_codex_redacts_working_tree_rows(bm_home: Path, tmp_path: Path) -> None: - # Regression: git status rows carry filenames/paths and must pass the same - # redaction floor as the transcript and cwd (a secret in a filename leaks - # otherwise). - project = tmp_path / "codex-proj" - (project / ".codex").mkdir(parents=True) - (project / ".codex" / "basic-memory.json").write_text( - json.dumps({"primaryProject": "demo"}), encoding="utf-8" + assert json.loads(second_stop.stdout) == {"continue": True} + assert checkpoint_requests.read("s-abc12345") is None + + +def test_codex_transcript_parser_reads_response_items_only(tmp_path: Path) -> None: + transcript = _codex_transcript(tmp_path) + + assert hook_module._transcript_turns(str(transcript), hook_module.Harness.codex) == [ + ("user", "Fix the login bug"), + ("assistant", "Found the null check issue"), + ] + + +def test_codex_stop_without_checkpoint_request_is_json_noop(bm_home: Path) -> None: + result = runner.invoke( + cli_app, + ["hook", "stop", "--harness", "codex"], + input=json.dumps({"session_id": "none", "stop_hook_active": False}), ) - transcript = _transcript(tmp_path) - mock_write = AsyncMock(return_value={"action": "created"}) - with ( - patch("basic_memory.mcp.tools.write_note", mock_write), - patch.object( - hook_module, "_git_status", return_value=["?? config/AKIAIOSFODNN7EXAMPLE.env"] - ), - ): - result = runner.invoke( - cli_app, - ["hook", "pre-compact", "--harness", "codex", "--project-dir", str(project)], - input=_payload(project, transcript_path=str(transcript), trigger="auto"), - ) assert result.exit_code == 0 - assert mock_write.await_args is not None - content = mock_write.await_args.kwargs["content"] - assert "## Working tree" in content - assert "AKIAIOSFODNN7EXAMPLE" not in content + assert json.loads(result.stdout) == {"continue": True} -def test_pre_compact_codex_skips_working_tree_when_workspace_denied( - bm_home: Path, tmp_path: Path -) -> None: - # Regression (#997): `git status --short` emits repo-relative filenames with - # no absolute prefix, so per-row redaction can't match a redactPaths entry. - # When the whole workspace is denied (cwd redacted to the marker), the - # section is skipped entirely rather than leaking the denied file list. - project = tmp_path / "codex-proj" - (project / ".codex").mkdir(parents=True) - (project / ".codex" / "basic-memory.json").write_text( - json.dumps({"primaryProject": "demo", "redactPaths": ["/srv/clients/"]}), - encoding="utf-8", +def test_codex_stop_clears_checkpoint_request_from_prior_turn(bm_home: Path) -> None: + from basic_memory.hooks import checkpoint_requests + + checkpoint_requests.create("s-abc12345", "turn-42") + + result = runner.invoke( + cli_app, + ["hook", "stop", "--harness", "codex"], + input=json.dumps( + { + "session_id": "s-abc12345", + "turn_id": "turn-43", + "stop_hook_active": False, + } + ), ) - transcript = _transcript(tmp_path) - mock_write = AsyncMock(return_value={"action": "created"}) - with ( - patch("basic_memory.mcp.tools.write_note", mock_write), - patch.object(hook_module, "_git_status", return_value=["M customer-roadmap.md"]), - ): - result = runner.invoke( - cli_app, - ["hook", "pre-compact", "--harness", "codex", "--project-dir", str(project)], - input=_payload( - "/srv/clients/acme/repo", transcript_path=str(transcript), trigger="auto" - ), - ) assert result.exit_code == 0 - assert mock_write.await_args is not None - kwargs = mock_write.await_args.kwargs - assert "## Working tree" not in kwargs["content"] - assert "customer-roadmap.md" not in kwargs["content"] - # The cwd itself is still redacted in frontmatter (the denial took effect). - assert kwargs["metadata"]["cwd"] == "[REDACTED_PATH]" + assert json.loads(result.stdout) == {"continue": True} + assert checkpoint_requests.read("s-abc12345") is None def test_pre_compact_codex_malformed_project_does_not_use_user_checkpoint_route( @@ -965,29 +985,27 @@ def test_hook_stdin_non_object_payload_normalizes(bm_home: Path, claude_project: # --- flush --- -def test_flush_reports_projector_summary(bm_home: Path) -> None: - from basic_memory.hooks.projector import FlushResult +def test_flush_reports_local_archive_summary(bm_home: Path) -> None: + from basic_memory.hooks.archive import FlushResult result_obj = FlushResult( swept=3, - projected=2, + archived=2, duplicates=1, pending=0, invalid=0, pruned=4, - notes=["Session s-1 (claude-code)"], ) with patch( - "basic_memory.hooks.projector.flush", new_callable=AsyncMock, return_value=result_obj + "basic_memory.hooks.archive.flush", new_callable=AsyncMock, return_value=result_obj ) as mock_flush: result = runner.invoke(cli_app, ["hook", "flush", "--older-than-days", "7"]) assert result.exit_code == 0 mock_flush.assert_awaited_once_with(older_than_days=7) - assert "swept 3 envelope(s): 2 projected, 1 duplicate(s), 0 pending, 0 invalid, 4 pruned" in ( + assert "swept 3 envelope(s): 2 archived, 1 duplicate(s), 0 pending, 0 invalid, 4 pruned" in ( result.stdout ) - assert "wrote: Session s-1 (claude-code)" in result.stdout def test_flush_rejects_negative_retention_window(bm_home: Path) -> None: @@ -1035,7 +1053,8 @@ def test_status_reports_inbox_and_settings( assert result.exit_code == 0 assert "pending envelopes: 1" in result.stdout - assert "processed envelopes: 1" in result.stdout + assert "archived envelopes: 1" in result.stdout + assert "pending checkpoint requests: 0" in result.stdout assert "last flush: 2026-07-15T10:00:00+00:00" in result.stdout assert "found" in result.stdout assert "primary project: demo" in result.stdout @@ -1113,6 +1132,9 @@ def test_install_codex_writes_hooks_json_with_matchers() -> None: "basic-memory hook session-start --harness codex" ) assert data["hooks"]["PreCompact"][0]["matcher"] == "manual|auto" + assert data["hooks"]["Stop"][0]["hooks"][0]["command"] == ( + "basic-memory hook stop --harness codex" + ) def test_install_preserves_existing_user_settings_and_hooks() -> None: @@ -1669,11 +1691,6 @@ def boom(*args, **kwargs): assert hook_module._uv_version() is None -def test_git_status_returns_empty_on_failure(tmp_path: Path) -> None: - # A directory that is not a git repo -> non-zero exit -> []. - assert hook_module._git_status(str(tmp_path)) == [] - - def test_claude_settings_precedence_and_local_overrides(tmp_path: Path) -> None: home = Path.home() # isolated_home points this at tmp_path (home / ".claude").mkdir(parents=True, exist_ok=True) diff --git a/tests/hooks/test_checkpoint_requests.py b/tests/hooks/test_checkpoint_requests.py new file mode 100644 index 000000000..7155258eb --- /dev/null +++ b/tests/hooks/test_checkpoint_requests.py @@ -0,0 +1,81 @@ +"""Tests for the private Codex PreCompact-to-Stop handshake state.""" + +import json +import os +import stat +from pathlib import Path + +import pytest + +from basic_memory.hooks import checkpoint_requests + + +def test_checkpoint_request_lifecycle_is_private(bm_home: Path) -> None: + request = checkpoint_requests.create("session-1", "turn-1") + + assert checkpoint_requests.pending_count() == 1 + assert checkpoint_requests.read("session-1") == request + + paths = list(checkpoint_requests.requests_dir().glob("*.json")) + assert len(paths) == 1 + if os.name != "nt": + assert stat.S_IMODE(checkpoint_requests.requests_dir().stat().st_mode) == 0o700 + assert stat.S_IMODE(paths[0].stat().st_mode) == 0o600 + + checkpoint_requests.clear("session-1") + assert checkpoint_requests.read("session-1") is None + assert checkpoint_requests.pending_count() == 0 + + +def test_checkpoint_request_replaces_same_session_atomically(bm_home: Path) -> None: + first = checkpoint_requests.create("session-1", "turn-1") + second = checkpoint_requests.create("session-1", "turn-2") + + assert second.requested_at >= first.requested_at + assert checkpoint_requests.read("session-1") == second + assert checkpoint_requests.pending_count() == 1 + assert not list(checkpoint_requests.requests_dir().glob("*.tmp")) + + +def test_checkpoint_request_requires_session_id(bm_home: Path) -> None: + with pytest.raises(ValueError, match="requires a Codex session id"): + checkpoint_requests.create("", None) + + +def test_checkpoint_request_rejects_non_object(bm_home: Path) -> None: + checkpoint_requests.create("session-1", None) + path = next(checkpoint_requests.requests_dir().glob("*.json")) + path.write_text("[]", encoding="utf-8") + + with pytest.raises(ValueError, match="invalid checkpoint request"): + checkpoint_requests.read("session-1") + + +def test_checkpoint_request_rejects_missing_fields(bm_home: Path) -> None: + checkpoint_requests.create("session-1", None) + path = next(checkpoint_requests.requests_dir().glob("*.json")) + path.write_text("{}", encoding="utf-8") + + with pytest.raises(ValueError, match="invalid checkpoint request"): + checkpoint_requests.read("session-1") + + +@pytest.mark.parametrize( + "override", + [ + {"source_session_id": "other"}, + {"source_turn_id": 42}, + {"requested_at": 42}, + ], +) +def test_checkpoint_request_rejects_invalid_field_types( + bm_home: Path, override: dict[str, object] +) -> None: + checkpoint_requests.create("session-1", None) + path = next(checkpoint_requests.requests_dir().glob("*.json")) + payload = json.loads(path.read_text(encoding="utf-8")) + payload.update(override) + path.write_text(json.dumps(payload), encoding="utf-8") + + with pytest.raises(ValueError, match="invalid checkpoint request"): + checkpoint_requests.read("session-1") diff --git a/tests/hooks/test_envelope.py b/tests/hooks/test_envelope.py index 9dc24be5c..2fcf06311 100644 --- a/tests/hooks/test_envelope.py +++ b/tests/hooks/test_envelope.py @@ -14,8 +14,6 @@ envelope_from_json, envelope_to_json, idempotency_key, - to_frontmatter_fields, - to_provenance_observations, ) @@ -122,60 +120,6 @@ def test_idempotency_key_is_metadata_only() -> None: assert plain.idempotency_key == secret.idempotency_key -# --- Projections --- - - -def test_provenance_observations_include_required_source() -> None: - envelope = _envelope(turn_id="turn-3") - - lines = to_provenance_observations(envelope) - - assert lines[0] == "- [source] claude-code/session-1" - assert f"- [event] {COMPACTION_IMMINENT} at 2026-07-15T10:00:00+00:00" in lines - assert f"- [idempotency] {envelope.idempotency_key}" in lines - assert "- [turn] turn-3" in lines - - -def test_provenance_observations_omit_turn_when_absent() -> None: - lines = to_provenance_observations(_envelope()) - - assert not any(line.startswith("- [turn]") for line in lines) - - -def test_provenance_observations_collapse_control_characters() -> None: - # Identity fields skip the payload redaction floor, so a newline-carrying - # id (hostile stdin or a corrupt replayed inbox file) must not become - # extra observation/relation lines in the projected note body. - envelope = _envelope( - session_id="s-1\n- [decision] injected", - turn_id="turn-1\r\n- relates_to [[Evil]]\x00done", - ) - - lines = to_provenance_observations(envelope) - - assert lines[0] == "- [source] claude-code/s-1 - [decision] injected" - assert "- [turn] turn-1 - relates_to [[Evil]] done" in lines - assert all("\n" not in line and "\r" not in line for line in lines) - - -def test_frontmatter_fields_are_queryable() -> None: - envelope = _envelope(turn_id="turn-3") - - fields = to_frontmatter_fields(envelope) - - assert fields == { - "envelope_id": envelope.id, - "envelope_source": "claude-code", - "envelope_event": COMPACTION_IMMINENT, - "idempotency_key": envelope.idempotency_key, - "envelope_turn_id": "turn-3", - } - - -def test_frontmatter_fields_omit_turn_when_absent() -> None: - assert "envelope_turn_id" not in to_frontmatter_fields(_envelope()) - - # --- Serialization --- @@ -187,7 +131,7 @@ def test_envelope_json_roundtrip() -> None: def test_envelope_from_json_rejects_non_object() -> None: # pydantic.ValidationError is a ValueError, so callers' existing handlers - # (projector flush, inbox prune gate) catch strict-validation failures too. + # (archive flush) catch strict-validation failures too. with pytest.raises(ValueError, match="should be an object"): envelope_from_json("[1, 2]") diff --git a/tests/hooks/test_projector.py b/tests/hooks/test_projector.py index 41b540e0d..08d2e1930 100644 --- a/tests/hooks/test_projector.py +++ b/tests/hooks/test_projector.py @@ -1,40 +1,31 @@ -"""Unit tests for the deterministic projector: dedup, replay-safety, mapping gate.""" +"""Tests for the local lifecycle-trace archive sweep.""" import json from pathlib import Path from unittest.mock import AsyncMock, patch from basic_memory.hooks import inbox -from basic_memory.hooks.envelope import ( - COMPACTION_IMMINENT, - SESSION_STARTED, - create_envelope, -) -from basic_memory.hooks.projector import _artifact_username, flush, split_project_ref - -WRITE_OK = {"title": "x", "action": "created"} +from basic_memory.hooks.envelope import SESSION_STARTED, create_envelope +from basic_memory.hooks.archive import flush +from basic_memory.hooks.project_ref import split_project_ref def _capture( + *, session_id: str = "s-1", - event: str = SESSION_STARTED, project_hint: str = "demo", ts: str = "2026-07-15T10:00:00+00:00", - source: str = "claude-code", - payload: dict | None = None, - turn_id: str | None = None, ) -> Path: - envelope = create_envelope( - source=source, - event=event, - session_id=session_id, - cwd="/tmp/workdir", - project_hint=project_hint, - ts=ts, - turn_id=turn_id, - payload=payload or {}, + return inbox.write_envelope( + create_envelope( + source="codex", + event=SESSION_STARTED, + session_id=session_id, + cwd="/tmp/workdir", + project_hint=project_hint, + ts=ts, + ) ) - return inbox.write_envelope(envelope) def test_split_project_ref_routes_uuids_via_project_id() -> None: @@ -45,385 +36,101 @@ def test_split_project_ref_routes_uuids_via_project_id() -> None: assert split_project_ref("my-team/notes") == ("my-team/notes", None) -def test_artifact_username_falls_back_to_environment() -> None: - with ( - patch("basic_memory.hooks.projector.getuser", side_effect=OSError), - patch.dict("os.environ", {"USER": "ci-user"}, clear=True), - ): - assert _artifact_username() == "ci-user" - - -def test_artifact_username_is_stable_when_unavailable() -> None: - with ( - patch("basic_memory.hooks.projector.getuser", side_effect=KeyError), - patch.dict("os.environ", {}, clear=True), - ): - assert _artifact_username() == "unknown" +async def test_flush_archives_trace_without_writing_graph_notes(bm_home: Path) -> None: + _capture(project_hint="") + _capture(session_id="s-2", project_hint="demo", ts="2026-07-15T10:01:00+00:00") + mock_write = AsyncMock() - -async def test_flush_projects_session_and_ledger(bm_home: Path) -> None: - _capture(event=SESSION_STARTED, ts="2026-07-15T10:00:00+00:00") - _capture(event=COMPACTION_IMMINENT, ts="2026-07-15T10:05:00+00:00") - mock_write = AsyncMock(return_value=WRITE_OK) - - with ( - patch("basic_memory.mcp.tools.write_note", mock_write), - patch("basic_memory.hooks.projector.getuser", return_value="alice"), - patch("basic_memory.hooks.projector.gethostname", return_value="devbox"), - ): + with patch("basic_memory.mcp.tools.write_note", mock_write): result = await flush() assert result.swept == 2 - assert result.projected == 2 + assert result.archived == 2 assert result.pending == 0 - assert result.notes == ["Session s-1 (claude-code)", "Tool Ledger s-1 (claude-code)"] + assert result.invalid == 0 + mock_write.assert_not_awaited() assert inbox.list_envelopes() == [] assert len(list(inbox.processed_dir().glob("*.json"))) == 2 assert inbox.last_flush() is not None - session_call = mock_write.await_args_list[0] - assert session_call.kwargs["project"] == "demo" - assert session_call.kwargs["overwrite"] is True - assert session_call.kwargs["directory"] == "sessions" - # Explicit note_type is what persists (content frontmatter type is stripped); - # without it the artifact lands as `note` and session recall can't find it. - assert session_call.kwargs["note_type"] == "session" - # Provenance rides metadata= so write_note serializes it safely (never a - # hand-built frontmatter block that an opaque field could invalidate). - session_metadata = session_call.kwargs["metadata"] - assert session_metadata["created_by"] == "bm-hook/claude-code" - assert session_metadata["caused_by_event"] - assert session_metadata["username"] == "alice" - assert session_metadata["hostname"] == "devbox" - assert session_metadata["status"] == "open" - content = session_call.kwargs["content"] - assert "- [source] claude-code/s-1" in content - assert "session_started at 2026-07-15T10:00:00+00:00" in content - assert "compaction_imminent at 2026-07-15T10:05:00+00:00" in content - - ledger_call = mock_write.await_args_list[1] - ledger_content = ledger_call.kwargs["content"] - assert ledger_call.kwargs["note_type"] == "tool_ledger" - assert ledger_call.kwargs["metadata"]["created_by"] == "bm-hook/claude-code" - assert ledger_call.kwargs["metadata"]["username"] == "alice" - assert ledger_call.kwargs["metadata"]["hostname"] == "devbox" - assert "- [event] session_started at" in ledger_content - assert "- [source] claude-code/s-1" in ledger_content - - -async def test_flush_passes_yaml_special_turn_id_through_metadata(bm_home: Path) -> None: - # source_turn_id is opaque, surface-defined text. A value with YAML-special - # content must reach write_note as a structured metadata value (serialized - # safely there), never hand-built into a `---` block that it would break — - # which would wedge the session pending on every flush retry. - _capture(event=SESSION_STARTED, turn_id="turn: 42\ninjected: true") - mock_write = AsyncMock(return_value=WRITE_OK) - - with patch("basic_memory.mcp.tools.write_note", mock_write): - result = await flush() - - # One envelope retired (projected counts envelopes); it wrote both artifacts. - assert result.projected == 1 - assert result.pending == 0 - session_call = mock_write.await_args_list[0] - assert session_call.kwargs["metadata"]["envelope_turn_id"] == "turn: 42\ninjected: true" - # The raw value is not spliced into a frontmatter block in the body. - assert "---" not in session_call.kwargs["content"] - - -async def test_flush_collapses_control_characters_in_note_bodies(bm_home: Path) -> None: - # Session ids are opaque, surface-defined text that skips the payload - # redaction floor. A newline-carrying id must not break out of its markdown - # line in EITHER artifact — titles, Events/Entries lines, and the ledger's - # [source] observation would otherwise parse as injected observations or - # relations in the projected note. - _capture(session_id="s-1\n- [decision] injected\n- relates_to [[Evil]]") - mock_write = AsyncMock(return_value=WRITE_OK) - - with patch("basic_memory.mcp.tools.write_note", mock_write): - result = await flush() - - assert result.projected == 1 - for call in mock_write.await_args_list: - content = call.kwargs["content"] - title = call.kwargs["title"] - assert "\n- [decision] injected" not in content - assert "\n- relates_to [[Evil]]" not in content - assert "\n" not in title - - -async def test_flush_skips_when_another_flush_holds_the_lock(bm_home: Path) -> None: - # Concurrent flushes race: a stale sweep can overwrite an artifact without a - # sibling's just-retired event. Holding the inbox lock, a second flush must - # skip entirely (write nothing, leave envelopes pending) rather than race. - _capture(event=SESSION_STARTED) - mock_write = AsyncMock(return_value=WRITE_OK) - - with inbox.flush_lock() as acquired: - assert acquired is True - with patch("basic_memory.mcp.tools.write_note", mock_write): - result = await flush() - assert result.skipped is True - assert result.swept == 0 - assert mock_write.await_count == 0 - # The envelope is untouched — the next unlocked flush still projects it. - assert len(inbox.list_envelopes()) == 1 - - -async def test_flush_uses_capture_folder_from_payload(bm_home: Path) -> None: - _capture(payload={"capture_folder": "codex-sessions"}) - mock_write = AsyncMock(return_value=WRITE_OK) - - with patch("basic_memory.mcp.tools.write_note", mock_write): - await flush() - - assert mock_write.await_args_list[0].kwargs["directory"] == "codex-sessions" - - -async def test_flush_routes_uuid_project_hints_via_project_id(bm_home: Path) -> None: - _capture(project_hint="0198f2b4-77aa-7bbf-9c2d-51e60a92d3c4") - mock_write = AsyncMock(return_value=WRITE_OK) - - with patch("basic_memory.mcp.tools.write_note", mock_write): - await flush() - - assert ( - mock_write.await_args_list[0].kwargs["project_id"] == "0198f2b4-77aa-7bbf-9c2d-51e60a92d3c4" - ) - - -async def test_flush_dedups_idempotency_replays_within_a_sweep(bm_home: Path) -> None: - # Same source/session/event/minute -> same idempotency key. +async def test_flush_deduplicates_replays_within_and_across_sweeps(bm_home: Path) -> None: _capture(ts="2026-07-15T10:00:01+00:00") _capture(ts="2026-07-15T10:00:41+00:00") - mock_write = AsyncMock(return_value=WRITE_OK) - with patch("basic_memory.mcp.tools.write_note", mock_write): - result = await flush() + first = await flush() + _capture(ts="2026-07-15T10:00:59+00:00") + second = await flush() - assert result.projected == 1 - assert result.duplicates == 1 - # The duplicate is retired, not re-projected: one session + one ledger write. - assert mock_write.await_count == 2 + assert first.archived == 1 + assert first.duplicates == 1 + assert second.archived == 0 + assert second.duplicates == 1 + assert len(list(inbox.processed_dir().glob("*.json"))) == 3 -async def test_flush_is_replay_safe_across_runs(bm_home: Path) -> None: - _capture(ts="2026-07-15T10:00:01+00:00") - mock_write = AsyncMock(return_value=WRITE_OK) +async def test_flush_counts_invalid_envelopes_and_leaves_them(bm_home: Path) -> None: + valid = _capture() + broken = valid.parent / f"{'0' * 8}-0000-7000-8000-{'0' * 12}.json" + broken.write_text("{not json", encoding="utf-8") - with patch("basic_memory.mcp.tools.write_note", mock_write): - first = await flush() - # The same hook replays after the first flush (same key, new envelope). - _capture(ts="2026-07-15T10:00:59+00:00") - second = await flush() + result = await flush() - assert first.projected == 1 - assert second.projected == 0 - assert second.duplicates == 1 - assert mock_write.await_count == 2 # no double-write for the replay + assert result.archived == 1 + assert result.invalid == 1 + assert broken.exists() -async def test_flush_leaves_unmapped_envelopes_pending(bm_home: Path) -> None: - path = _capture(project_hint="") - mock_write = AsyncMock(return_value=WRITE_OK) +async def test_flush_counts_type_corrupt_envelope_and_keeps_sweeping(bm_home: Path) -> None: + valid = _capture() + corrupt = valid.parent / f"{'1' * 8}-0000-7000-8000-{'1' * 12}.json" + payload = json.loads(valid.read_text(encoding="utf-8")) + payload["source_session_id"] = [] + corrupt.write_text(json.dumps(payload), encoding="utf-8") - with patch("basic_memory.mcp.tools.write_note", mock_write): - result = await flush() + result = await flush() - assert result.pending == 1 - assert result.projected == 0 - mock_write.assert_not_awaited() - assert inbox.list_envelopes() == [path] # still pending, self-heals later + assert result.archived == 1 + assert result.invalid == 1 + assert corrupt.exists() -async def test_flush_leaves_group_pending_when_write_fails(bm_home: Path) -> None: +async def test_flush_leaves_envelope_pending_when_local_archive_fails(bm_home: Path) -> None: path = _capture() - mock_write = AsyncMock(side_effect=RuntimeError("api down")) - with patch("basic_memory.mcp.tools.write_note", mock_write): + with patch.object(inbox, "mark_processed", side_effect=OSError("disk full")): result = await flush() + assert result.archived == 0 assert result.pending == 1 - assert result.projected == 0 assert inbox.list_envelopes() == [path] -async def test_flush_routes_on_project_hint_from_in_group_replay(bm_home: Path) -> None: - # Same-minute same-event replay where the mapping was set only for the later - # capture: the first envelope has no project_hint, the replay carries "demo". - # Routing must use the replay's hint, not leave the group pending (and prune). - _capture(event=SESSION_STARTED, ts="2026-07-15T10:00:01+00:00", project_hint="") - _capture(event=SESSION_STARTED, ts="2026-07-15T10:00:41+00:00", project_hint="demo") - mock_write = AsyncMock(return_value=WRITE_OK) - - with patch("basic_memory.mcp.tools.write_note", mock_write): - result = await flush() - - assert result.projected == 1 - assert result.pending == 0 - assert result.duplicates == 1 - assert mock_write.await_args_list[0].kwargs["project"] == "demo" - assert inbox.list_envelopes() == [] - - -async def test_flush_routes_new_event_via_processed_replay_hint(bm_home: Path) -> None: - # Sweep 1 retires a no-hint original AND its hinted same-key replay into - # processed/. A later distinct event for the same session must still route: - # _dedup_by_key keeps the earlier (hint-less) processed envelope for the - # rendered rows, so routing must scan the full processed history (which holds - # the replay's hint), or the new event stays pending forever. - mock_write = AsyncMock(return_value=WRITE_OK) - with patch("basic_memory.mcp.tools.write_note", mock_write): - _capture(event=SESSION_STARTED, ts="2026-07-15T10:00:01+00:00", project_hint="") - _capture(event=SESSION_STARTED, ts="2026-07-15T10:00:41+00:00", project_hint="demo") - first = await flush() - _capture(event=COMPACTION_IMMINENT, ts="2026-07-15T10:05:00+00:00", project_hint="") - second = await flush() - - assert first.projected == 1 - assert second.projected == 1 # routed via the processed replay's hint - assert second.pending == 0 - assert inbox.list_envelopes() == [] - - -async def test_flush_dedups_retired_replay_history_on_rebuild(bm_home: Path) -> None: - # A same-minute SessionStart replay is retired into processed/ alongside its - # original (same key, distinct id). When a later event triggers a full - # rebuild, the replay must not resurface as a duplicate row. - _capture(event=SESSION_STARTED, ts="2026-07-15T10:00:01+00:00") - _capture(event=SESSION_STARTED, ts="2026-07-15T10:00:41+00:00") # replay, same key - mock_write = AsyncMock(return_value=WRITE_OK) - - with patch("basic_memory.mcp.tools.write_note", mock_write): - first = await flush() - _capture(event=COMPACTION_IMMINENT, ts="2026-07-15T10:05:00+00:00") - second = await flush() - - assert first.projected == 1 - assert first.duplicates == 1 - assert second.projected == 1 - - session_writes = [ - call for call in mock_write.await_args_list if call.kwargs["title"].startswith("Session ") - ] - rebuilt = session_writes[-1].kwargs["content"] - # Exactly one session_started event row (the "- at" Events line), - # despite the retired replay sitting in processed/. - assert rebuilt.count("- session_started at") == 1 - assert "- compaction_imminent at" in rebuilt - - -async def test_flush_reprojects_after_write_failure_with_in_group_replay(bm_home: Path) -> None: - # Two same-minute captures share one idempotency key: one fresh, one in-group - # replay. If the write fails, retiring the replay early would let the next - # sweep read its key from processed/ and wrongly retire the still-unprojected - # fresh envelope — the session would never land. Both must stay pending and - # project together on the healed sweep. - _capture(ts="2026-07-15T10:00:01+00:00") - _capture(ts="2026-07-15T10:00:41+00:00") # same key (minute granularity) - - failing = AsyncMock(side_effect=RuntimeError("api down")) - with patch("basic_memory.mcp.tools.write_note", failing): - first = await flush() - - assert first.projected == 0 - assert first.pending == 1 - # Nothing retired — the in-group replay must not have moved to processed/. - assert list(inbox.processed_dir().glob("*.json")) == [] - assert len(inbox.list_envelopes()) == 2 - - healthy = AsyncMock(return_value=WRITE_OK) - with patch("basic_memory.mcp.tools.write_note", healthy): - second = await flush() - - assert second.projected == 1 - assert second.duplicates == 1 - assert healthy.await_count == 2 # one session + one ledger, no double-write - assert inbox.list_envelopes() == [] - - -async def test_flush_leaves_group_pending_on_error_result(bm_home: Path) -> None: - path = _capture() - mock_write = AsyncMock(return_value={"error": "NOTE_WRITE_BLOCKED"}) +async def test_flush_skips_when_another_flush_holds_the_lock(bm_home: Path) -> None: + _capture() - with patch("basic_memory.mcp.tools.write_note", mock_write): + with inbox.flush_lock() as acquired: + assert acquired is True result = await flush() - assert result.pending == 1 - assert inbox.list_envelopes() == [path] + assert result.skipped is True + assert result.swept == 0 + assert len(inbox.list_envelopes()) == 1 async def test_flush_skips_vanished_inbox_file(bm_home: Path) -> None: - # A pending file that disappears (a concurrent flush retired it) or is - # transiently unreadable between listing and reading must not abort the - # sweep: the valid envelope still projects and the flush marker is recorded. good = _capture() - ghost = inbox.inbox_dir() / "gone.json" # listed but never on disk - mock_write = AsyncMock(return_value=WRITE_OK) + ghost = inbox.inbox_dir() / "gone.json" - with ( - patch("basic_memory.mcp.tools.write_note", mock_write), - patch.object(inbox, "list_envelopes", return_value=[good, ghost]), - ): + with patch.object(inbox, "list_envelopes", return_value=[good, ghost]): result = await flush() - assert result.projected == 1 + assert result.archived == 1 assert result.swept == 2 - assert result.invalid == 0 # a vanished file is skipped, not counted corrupt - assert inbox.last_flush() is not None - - -async def test_flush_counts_invalid_envelopes_and_leaves_them(bm_home: Path) -> None: - valid = _capture() - broken = valid.parent / f"{'0' * 8}-0000-7000-8000-{'0' * 12}.json" - broken.write_text("{not json", encoding="utf-8") - mock_write = AsyncMock(return_value=WRITE_OK) - - with patch("basic_memory.mcp.tools.write_note", mock_write): - result = await flush() - - assert result.invalid == 1 - assert result.projected == 1 - assert broken.exists() # never deleted, never guessed at - - -async def test_flush_counts_type_corrupt_envelope_and_keeps_sweeping(bm_home: Path) -> None: - # Valid JSON, right keys, wrong scalar type (session id is a list). Must be - # counted invalid — not crash the sweep grouping on an unhashable id — so the - # valid envelope still projects. - valid = _capture() - corrupt = valid.parent / f"{'0' * 8}-0000-7000-8000-{'0' * 12}.json" - payload = json.loads(valid.read_text(encoding="utf-8")) - payload["source_session_id"] = [] - corrupt.write_text(json.dumps(payload), encoding="utf-8") - mock_write = AsyncMock(return_value=WRITE_OK) - - with patch("basic_memory.mcp.tools.write_note", mock_write): - result = await flush() - - assert result.invalid == 1 - assert result.projected == 1 - assert corrupt.exists() # left in place for inspection - - -async def test_flush_groups_sessions_independently(bm_home: Path) -> None: - _capture(session_id="s-1") - _capture(session_id="s-2", source="codex") - mock_write = AsyncMock(return_value=WRITE_OK) - - with patch("basic_memory.mcp.tools.write_note", mock_write): - result = await flush() - - assert result.projected == 2 - titles = sorted(result.notes) - assert "Session s-1 (claude-code)" in titles - assert "Session s-2 (codex)" in titles - assert mock_write.await_count == 4 # two artifacts per session group + assert result.invalid == 0 def _uuid7_aged(days_old: int): - """A uuid7 whose embedded timestamp is ``days_old`` days in the past.""" import uuid from datetime import datetime, timedelta, timezone @@ -435,180 +142,16 @@ def _uuid7_aged(days_old: int): return uuid.UUID(int=value) -def _plant_processed_with_age(days_old: int) -> Path: - """Plant a processed envelope whose uuid7 filename encodes a capture age.""" +async def test_flush_prunes_only_expired_archived_trace(bm_home: Path) -> None: directory = inbox.processed_dir() directory.mkdir(parents=True, exist_ok=True) - path = directory / f"{_uuid7_aged(days_old)}.json" - path.write_text("{}", encoding="utf-8") - return path - - -def _plant_pending_with_age( - days_old: int, - project_hint: str = "", - *, - event: str = SESSION_STARTED, - session_id: str = "stale", - ts: str = "2026-07-15T10:00:00+00:00", -) -> Path: - """Plant a pending inbox envelope with a valid body and an aged filename.""" - from basic_memory.hooks.envelope import envelope_to_json - - envelope = create_envelope( - source="claude-code", - event=event, - session_id=session_id, - cwd="/tmp/workdir", - project_hint=project_hint, - ts=ts, - ) - directory = inbox.inbox_dir() - directory.mkdir(parents=True, exist_ok=True) - path = directory / f"{_uuid7_aged(days_old)}.json" - path.write_text(envelope_to_json(envelope), encoding="utf-8") - return path + stale = directory / f"{_uuid7_aged(45)}.json" + stale.write_text("{}", encoding="utf-8") + pending = _capture(project_hint="") - -async def test_flush_prunes_expired_processed_envelopes(bm_home: Path) -> None: - _plant_processed_with_age(days_old=45) - mock_write = AsyncMock(return_value=WRITE_OK) - - with patch("basic_memory.mcp.tools.write_note", mock_write): - result = await flush() - - assert result.pruned == 1 - - -async def test_flush_prunes_expired_unresolved_pending_envelopes(bm_home: Path) -> None: - # A fully-unmapped session (no project_hint ever) would otherwise sit pending - # forever; retention bounds the inbox the same way it bounds processed/. - stale = _plant_pending_with_age(days_old=45) - mock_write = AsyncMock(return_value=WRITE_OK) - - with patch("basic_memory.mcp.tools.write_note", mock_write): - result = await flush() + result = await flush(older_than_days=30) assert result.pruned == 1 assert not stale.exists() - mock_write.assert_not_awaited() - - -async def test_flush_prune_skips_non_file_glob_matches(bm_home: Path) -> None: - # A directory whose name happens to match *.json must be skipped by - # retention, not unlinked (which would crash the sweep). - stray = inbox.inbox_dir() / f"{_uuid7_aged(45)}.json" - stray.mkdir(parents=True) - mock_write = AsyncMock(return_value=WRITE_OK) - - with patch("basic_memory.mcp.tools.write_note", mock_write): - result = await flush() - - assert stray.is_dir() # never unlinked - assert result.pruned == 0 - - -async def test_flush_keeps_recent_unmapped_pending_envelopes(bm_home: Path) -> None: - # Within the window, unmapped trace is kept — a mapping may still resolve it. - fresh = _plant_pending_with_age(days_old=1) - mock_write = AsyncMock(return_value=WRITE_OK) - - with patch("basic_memory.mcp.tools.write_note", mock_write): - result = await flush() - - assert result.pruned == 0 - assert fresh.exists() - - -async def test_flush_keeps_hintless_envelope_when_session_is_routable(bm_home: Path) -> None: - # e1 captured before primaryProject was set (hint-less), e2 after (hinted), - # same session, both past retention, and the write still fails (outage). The - # session is routable via e2, so retention must keep BOTH — pruning the - # hint-less e1 would drop its event from the session the next successful - # sweep rebuilds. - e1 = _plant_pending_with_age( - days_old=46, project_hint="", event=SESSION_STARTED, session_id="s-mixed" - ) - e2 = _plant_pending_with_age( - days_old=45, - project_hint="demo", - event=COMPACTION_IMMINENT, - session_id="s-mixed", - ts="2026-07-15T10:05:00+00:00", - ) - failing = AsyncMock(side_effect=RuntimeError("cloud down")) - - with patch("basic_memory.mcp.tools.write_note", failing): - result = await flush() - - assert e1.exists() # hint-less, but its session is routable via e2 - assert e2.exists() - assert result.pruned == 0 - - -async def test_flush_keeps_write_failed_mapped_envelope_past_retention(bm_home: Path) -> None: - # An old pending envelope WITH a project hint is pending because a write - # failed (cloud/API outage), not because it's unmappable. Retention must not - # drop it — that would defeat the self-heal path; only unresolvable - # (hint-less) trace is pruned. - old = _plant_pending_with_age(days_old=45, project_hint="demo") - failing = AsyncMock(side_effect=RuntimeError("cloud down")) - - with patch("basic_memory.mcp.tools.write_note", failing): - result = await flush() - - assert old.exists() # kept despite being past retention - assert result.pruned == 0 - assert result.pending == 1 - - -async def test_flush_ignores_unreadable_processed_files_for_dedup(bm_home: Path) -> None: - directory = inbox.processed_dir() - directory.mkdir(parents=True, exist_ok=True) - (directory / "junk.json").write_text("{not json", encoding="utf-8") - _capture() - mock_write = AsyncMock(return_value=WRITE_OK) - - with patch("basic_memory.mcp.tools.write_note", mock_write): - result = await flush() - - assert result.projected == 1 - - -async def test_flush_preserves_prior_events_on_incremental_flush(bm_home: Path) -> None: - """Regression: a later sweep must not erase events an earlier sweep projected. - - session_started is projected and retired to processed/ on flush 1; when - compaction_imminent arrives, flush 2 must rebuild the note from the full - session, not just the pending envelope. Overwriting from fresh alone would - drop session_started. - """ - mock_write = AsyncMock(return_value=WRITE_OK) - with patch("basic_memory.mcp.tools.write_note", mock_write): - _capture(event=SESSION_STARTED, ts="2026-07-15T10:00:00+00:00") - await flush() - _capture(event=COMPACTION_IMMINENT, ts="2026-07-15T10:30:00+00:00") - await flush() - - last_session = [ - call for call in mock_write.await_args_list if call.kwargs["title"].startswith("Session ") - ][-1] - content = last_session.kwargs["content"] - assert "session_started at 2026-07-15T10:00:00+00:00" in content - assert "compaction_imminent at 2026-07-15T10:30:00+00:00" in content - - -async def test_flush_distinguishes_sessions_sharing_a_prefix(bm_home: Path) -> None: - """Regression: sessions sharing an 8-char prefix get distinct titles/permalinks.""" - _capture(session_id="abcd1234-session-one") - _capture(session_id="abcd1234-session-two") - mock_write = AsyncMock(return_value=WRITE_OK) - - with patch("basic_memory.mcp.tools.write_note", mock_write): - result = await flush() - - session_titles = {title for title in result.notes if title.startswith("Session ")} - assert session_titles == { - "Session abcd1234-session-one (claude-code)", - "Session abcd1234-session-two (claude-code)", - } + # Project mapping is irrelevant now: valid pending trace is archived, not pruned. + assert not pending.exists() diff --git a/tests/test_codex_plugin_package.py b/tests/test_codex_plugin_package.py index ecbc546ed..044f0b36f 100644 --- a/tests/test_codex_plugin_package.py +++ b/tests/test_codex_plugin_package.py @@ -38,6 +38,7 @@ def test_codex_plugin_hooks_are_zero_logic_uv_scripts() -> None: for script, verb in ( ("session_start.py", "session-start"), ("pre_compact.py", "pre-compact"), + ("stop.py", "stop"), ): text = (hooks_dir / script).read_text(encoding="utf-8") assert "# /// script" in text @@ -58,6 +59,7 @@ def test_release_recipes_pin_codex_hooks_to_the_release_tag() -> None: assert justfile.count('just set-codex-hook-version "{{version}}"') == 2 assert 'just set-codex-hook-version "$(git rev-parse HEAD)"' not in justfile + assert "uv add --script plugins/codex/hooks/stop.py" in justfile def test_codex_plugin_marketplace_identity() -> None: @@ -79,6 +81,8 @@ def test_codex_plugin_docs_explain_global_install_and_repo_mapping() -> None: assert 'codex plugin marketplace add "$(git rev-parse --show-toplevel)"' in readme assert "codex plugin add codex@basic-memory" in readme assert "Plugin installation is user-level in Codex" in readme + assert "Selecting only `plugins/codex` omits" in readme + assert "marketplace file should not" in readme assert "Configuration can live at user level in `~/.codex/basic-memory.json`" in readme assert "the nearest project file overrides only the keys it declares" in readme assert "keep both the profile and checkout-specific repository" in readme @@ -153,6 +157,20 @@ def test_bm_checkpoint_tells_a_story_and_uses_graph_semantics() -> None: assert "Do not invent intent, impact, verification, decisions, or drama" in writing +def test_codex_checkpoint_applies_accumulated_redaction_before_write() -> None: + repo_root = Path(__file__).resolve().parents[1] + skill = (repo_root / "plugins/codex/skills/bm-checkpoint/SKILL.md").read_text(encoding="utf-8") + + assert "## Privacy Gate" in skill + assert "`redactKeys` and `redactPaths` accumulate" in skill + assert "Scrub **every string** passed to `write_note`" in skill + assert "Do not omit schema-required path fields; use the marker" in skill + assert "[REDACTED_PATH]" in skill + assert "[REDACTED]" in skill + assert "fail closed: skip the checkpoint" in skill + assert "Do not fall back to an unredacted note" in skill + + def test_infographics_skill_keeps_weekly_contract_and_bm_style_pool() -> None: repo_root = Path(__file__).resolve().parents[1] skill = (repo_root / ".agents/skills/infographics/SKILL.md").read_text(encoding="utf-8")