diff --git a/.env.example b/.env.example index 9b96ac5..d2811e3 100644 --- a/.env.example +++ b/.env.example @@ -34,6 +34,16 @@ ENGRAPHIS_API_TOKEN= # checkout or the platform user-data directory from an installed wheel. # ENGRAPHIS_DB_PATH=./engraphis.db +# Code indexing accepts roots under the working, home, or temporary directories by +# default. Set this to replace those defaults with an explicit absolute-path allow-list. +# Separate paths with ";" on Windows or ":" on macOS/Linux. +# ENGRAPHIS_INDEX_ROOTS=/srv/engraphis/repos:/mnt/code +# The dashboard and REST /api/code/index route accept paths only below this one root. +# An explicit root (or the ENGRAPHIS_INDEX_ROOTS fallback entry) must be absolute. +# Defaults to the first ENGRAPHIS_INDEX_ROOTS entry, or the current directory; an +# explicit HTTP root is included in the engine-approved set. +# ENGRAPHIS_HTTP_INDEX_ROOT=/srv/engraphis/repos + # ── Encryption at rest (optional; SQLCipher / AES-256) ────────────────────── # Transparently encrypt the ENTIRE memory database file. OFF by default. Needs the extra: # pip install "engraphis[encryption]" diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 02d53ae..ee63b14 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -16,6 +16,11 @@ jobs: analyze: name: Analyze ${{ matrix.language }} runs-on: ubuntu-latest + # Pull-request analyses otherwise use diff-informed queries, whose SARIF + # intentionally omits findings outside changed lines. The release gate + # must inspect complete raw SARIF on both pull requests and pushes. + env: + CODEQL_ACTION_DIFF_INFORMED_QUERIES: "false" strategy: fail-fast: false matrix: diff --git a/CHANGELOG.md b/CHANGELOG.md index aa76db4..22edb18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -71,7 +71,8 @@ Public 1.0.1 client reliability release. - API error responses and provider logs no longer expose arbitrary exception or configuration text; local folder and repository reads resolve and re-check filesystem boundaries. - Entity extraction and dashboard asset migration avoid adversarial regular-expression - backtracking, and CI now fails when CodeQL emits any SARIF finding. + backtracking. CodeQL now disables pull-request diff-informed analysis and CI fails on every + raw SARIF result, including pre-existing and source-suppressed results. ## [1.0.0] - 2026-07-23 diff --git a/README.md b/README.md index 2d0b6e9..b336093 100644 --- a/README.md +++ b/README.md @@ -632,6 +632,8 @@ All via environment (or `.env`): | `ENGRAPHIS_API_TOKEN` | — | Optional bearer credential for this single-user local customer node; never reuse a hosted credential | | `ENGRAPHIS_CORS_ORIGINS` | loopback on `ENGRAPHIS_PORT` | Comma-separated REST CORS allow-list; defaults to `127.0.0.1` and `localhost` on the configured port | | `ENGRAPHIS_WORKSPACES` | — | Optional comma-separated server-side workspace allow-list | +| `ENGRAPHIS_INDEX_ROOTS` | Working, home, and temporary directories | Optional path-separator-delimited absolute-path allow-list that replaces the default roots accepted by local code indexing | +| `ENGRAPHIS_HTTP_INDEX_ROOT` | First `ENGRAPHIS_INDEX_ROOTS` entry, or current directory | Single root for dashboard and REST `POST /api/code/index`; submitted paths resolve beneath it. An explicit root (or fallback entry) must be absolute; an explicit HTTP root is included in the engine-approved set. MCP and CLI indexing continue to use `ENGRAPHIS_INDEX_ROOTS`. | | `ENGRAPHIS_DB_KEY` | — | Encrypt the database at rest (SQLCipher). Or use `ENGRAPHIS_DB_KEY_FILE` | | `ENGRAPHIS_EMBED_MODEL` | `sentence-transformers/all-MiniLM-L6-v2` | sentence-transformers model | | `ENGRAPHIS_EXTRACTOR` | `none` | `none` = verbatim; `chunk` = offline structure-aware chunks; `llm` = free-form LLM facts; `llm_structured` = schema-validated facts + graph metadata | diff --git a/SECURITY.md b/SECURITY.md index 028d384..ddeb5d6 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -59,9 +59,18 @@ DOMPurify at all render sites. Verified against payloads with `onerror` handlers ### 5. Code indexing `engraphis_index_repo` parses source files under a path you give it — same trust boundary as any other local tool the agent has. Path is attacker-controlled if agent's instructions are. -`max_files`/`max_file_bytes` bound resource use, not access scope. Traversal does not follow -file symlinks outside the root, prunes dependency/build directories during the walk, and honors -the root `.engraphisignore` without allowing negation rules to re-expose hardcoded excludes. +Canonical roots are restricted to the working, home, or temporary directories by default. +Set `ENGRAPHIS_INDEX_ROOTS` to a path-separator-delimited absolute-path operator allow-list to +replace those defaults for nonstandard mounts or a narrower deployment boundary. +Dashboard and REST `POST /api/code/index` use the stricter single-root boundary +`ENGRAPHIS_HTTP_INDEX_ROOT`: submitted paths resolve beneath that root. It defaults to the first +`ENGRAPHIS_INDEX_ROOTS` entry, or the current directory. An explicit HTTP root (or fallback +entry) must be absolute; an explicit HTTP root is included in the engine-approved set. MCP and +CLI indexing retain the `ENGRAPHIS_INDEX_ROOTS` allow-list semantics. +`max_files`/`max_file_bytes` bound resource use, not access within an allowed root. Traversal +does not follow file symlinks outside the root, prunes dependency/build directories during the +walk, and honors the root `.engraphisignore` without allowing negation rules to re-expose +hardcoded excludes. Anyone who can reach an authenticated local mutation route has the authority of that local installation, so do not share its bearer token. diff --git a/engraphis/backends/embedder_deterministic.py b/engraphis/backends/embedder_deterministic.py index ff3fa49..f564712 100644 --- a/engraphis/backends/embedder_deterministic.py +++ b/engraphis/backends/embedder_deterministic.py @@ -28,13 +28,12 @@ def dim(self) -> int: def embed(self, texts: list[str], *, kind: Literal["text", "code"] = "text") -> np.ndarray: out = np.zeros((len(texts), self._dim), dtype=np.float32) for i, text in enumerate(texts): - for token in _tokenize(text, kind): + for feature in _tokenize(text, kind): # SHA-1 is retained only to keep the established offline embedding # mapping stable across upgrades. This is feature hashing, never a # password, signature, integrity check, or other security primitive. - # codeql[py/weak-sensitive-data-hashing] h = hashlib.sha1( - token.encode("utf-8"), usedforsecurity=False + feature.encode("utf-8"), usedforsecurity=False ).digest() idx = int.from_bytes(h[:4], "big") % self._dim sign = 1.0 if h[4] & 1 else -1.0 diff --git a/engraphis/core/engine.py b/engraphis/core/engine.py index 8ca802e..a3b20de 100644 --- a/engraphis/core/engine.py +++ b/engraphis/core/engine.py @@ -14,7 +14,9 @@ import json import logging import math +import os import re +import tempfile import threading import time from collections import defaultdict, deque @@ -78,6 +80,44 @@ CODE_EXPORT_MAX_LIMIT = 20_000 +def _approved_local_index_roots() -> tuple[str, ...]: + """Return canonical roots available to the explicit local indexing capability. + + ``ENGRAPHIS_INDEX_ROOTS`` is an operator-owned, path-separator-delimited allow-list. + ``ENGRAPHIS_HTTP_INDEX_ROOT`` is a separately configured, single HTTP boundary; + include it here too so the checked path handed off by the HTTP route remains usable + by the engine. Configured paths must be absolute: silently resolving a relative + operator setting against the process working directory would make the boundary + deployment-dependent. + When it is unset, the working directory, home directory, and system temporary + directory preserve the local-first defaults used by ordinary agent/project checkouts. + """ + def canonical_configured_root(value: str, setting: str) -> str: + if not os.path.isabs(value): + raise ValueError(f"{setting} must contain only absolute paths") + return os.path.normcase(os.path.realpath(os.path.expanduser(value))) + + configured = [ + canonical_configured_root(value.strip(), "ENGRAPHIS_INDEX_ROOTS") + for value in os.environ.get("ENGRAPHIS_INDEX_ROOTS", "").split(os.pathsep) + if value.strip() + ] + if configured: + roots = configured + else: + roots = [ + os.path.normcase(os.path.realpath(os.getcwd())), + os.path.normcase(os.path.realpath(os.path.expanduser("~"))), + os.path.normcase(os.path.realpath(tempfile.gettempdir())), + ] + + http_root = os.environ.get("ENGRAPHIS_HTTP_INDEX_ROOT", "").strip() + if http_root: + roots.append(canonical_configured_root(http_root, "ENGRAPHIS_HTTP_INDEX_ROOT")) + + return tuple(dict.fromkeys(roots)) + + def _bounded_finite(value, *, default: float, minimum: float, maximum: float) -> float: try: number = float(value) @@ -1228,11 +1268,32 @@ def index_repo(self, repo_id: str, root_path: str, *, languages: Optional[set] = ) indexer = get_code_indexer(prefer=prefer) - # index_repo is an explicit local-filesystem capability: callers select the - # repository root to index. Every walked child is resolved and re-contained - # below before it is read. - # codeql[py/path-injection] - root = Path(root_path).expanduser().resolve() + # index_repo is an explicit local-filesystem capability: callers select a + # repository in one of the approved local roots. Canonicalizing then checking + # containment confines the capability to the operator's configured/default + # filesystem boundary. Every walked child is re-contained below before it is read. + # Normalize before the root-prefix check. Keep this check at the filesystem + # call site (rather than hiding it in a helper) so both human review and static + # analysis can establish that every later path is constrained by this boundary. + canonical_root = os.path.normcase( + os.path.realpath(os.path.expanduser(os.fspath(root_path))) + ) + # Retain a separator on the checked value. This makes an approved root + # itself and every child use the same normalized-prefix check (and avoids + # accepting a sibling such as ``/work/repo-copy`` for ``/work/repo``). + # It also keeps the checked value, rather than the untrusted input, as + # the only path passed to filesystem APIs below. + canonical_root_with_sep = canonical_root.rstrip(os.sep) + os.sep + safe_root: Optional[str] = None + for approved_root in _approved_local_index_roots(): + normalized_approved = os.path.normcase(os.path.realpath(approved_root)) + approved_prefix = normalized_approved.rstrip(os.sep) + os.sep + if canonical_root_with_sep.startswith(approved_prefix): + safe_root = canonical_root_with_sep + break + if safe_root is None: + raise ValueError("repo root is outside approved local roots") + root = Path(safe_root) if not root.exists(): raise ValueError(f"repo root not found: {root_path}") if not root.is_dir(): diff --git a/engraphis/mcp_server.py b/engraphis/mcp_server.py index 294fb5d..78df7bc 100644 --- a/engraphis/mcp_server.py +++ b/engraphis/mcp_server.py @@ -715,8 +715,8 @@ def engraphis_index_repo( repo: Annotated[str, Field(description="Repo name to index.", min_length=1, max_length=200)], root_path: Annotated[str, Field(description="Local filesystem path to the repo root " - "to parse (e.g. '/home/user/projects/myrepo'). Reads files from " - "this path the same way any local tool you have would.", + "to parse (e.g. '/home/user/projects/myrepo'). The path must be " + "inside the local defaults or ENGRAPHIS_INDEX_ROOTS allow-list.", min_length=1, max_length=4_000)], languages: Annotated[Optional[List[str]], Field(description="Restrict to these " "languages (e.g. ['python','csharp']). Names are normalised " @@ -739,7 +739,9 @@ def engraphis_index_repo( engraphis_remember). Re-indexing is safe to call again; each file's symbols are replaced, not duplicated. Reads files from ``root_path`` on the local filesystem — the same trust boundary as any other local tool you have, nothing is sent anywhere. - Each completed scan appends a fresh operation receipt, so the MCP call is + Set ``ENGRAPHIS_INDEX_ROOTS`` to a path-separator-delimited absolute-path allow-list when + repositories live outside the working, home, or temporary directories, or to narrow the + defaults. Each completed scan appends a fresh operation receipt, so the MCP call is non-idempotent even when the code graph itself is unchanged. Returns: diff --git a/engraphis/routes/v2_api.py b/engraphis/routes/v2_api.py index 4cddbf5..d8b6543 100644 --- a/engraphis/routes/v2_api.py +++ b/engraphis/routes/v2_api.py @@ -35,6 +35,40 @@ _service: Optional[MemoryService] = None +def _invalid_request() -> HTTPException: + """Return the stable public boundary for service-layer validation failures. + + ``ValidationError`` can be raised after processing client-controlled names, ids, and + paths. Its message is useful for local diagnostics, but is not a safe API contract: + it could echo an untrusted value or a future implementation detail. Keep the HTTP + response deliberately fixed and never log the error text at API call sites. + """ + return HTTPException(status_code=400, detail={"error": "invalid request"}) + + +class _HttpCodeIndexConfigurationError(RuntimeError): + """Raised when the operator's HTTP indexing boundary is unsafe.""" + + +def _http_index_configuration_error() -> HTTPException: + """Keep operator configuration failures distinct from invalid client paths.""" + return HTTPException(status_code=500, detail={"error": "internal server error"}) + + +def _sanitized_http_exception(status_code: object) -> HTTPException: + """Keep a downstream HTTP status without propagating its detail or traceback. + + A service dependency can deliberately signal an HTTP status, but its exception + object and detail are not part of this route's public contract. Preserve a valid + error-class status while returning a route-owned, static body. + """ + if isinstance(status_code, int) and 400 <= status_code <= 599: + if status_code < 500: + return HTTPException(status_code=status_code, detail={"error": "request rejected"}) + return HTTPException(status_code=status_code, detail={"error": "internal server error"}) + return HTTPException(status_code=500, detail={"error": "internal server error"}) + + def service() -> MemoryService: """Lazily bind a single MemoryService to the configured store (the live v2 DB).""" global _service @@ -86,17 +120,14 @@ def _run(fn, *a, **k): "limit": exc.limit, "recommended_action": "narrow repository, time, type, or relation filters", }) from None - except ValidationError as exc: - logger.info("dashboard request rejected (%s)", type(exc).__name__) - # ValidationError is the service layer's deliberately public, bounded input-error - # type. It is not a traceback or arbitrary provider exception. - # codeql[py/stack-trace-exposure] - raise HTTPException(status_code=400, detail={"error": str(exc)}) from None - except HTTPException: - raise + except ValidationError: + logger.info("dashboard request rejected") + raise _invalid_request() from None + except HTTPException as exc: + logger.info("dashboard dependency rejected request (status=%s)", exc.status_code) + raise _sanitized_http_exception(exc.status_code) from None except Exception as exc: # noqa: BLE001 - msg = str(exc) - if "not aligned" in msg or ("256" in msg and "384" in msg): + if _is_embedder_mismatch(exc): raise HTTPException(status_code=409, detail={ "error": "Semantic search needs the embedding model that built your data " "(sentence-transformers / all-MiniLM). Install it once — " @@ -167,8 +198,9 @@ def _mem(m: dict) -> dict: def _is_embedder_mismatch(exc) -> bool: - msg = str(exc) - return "not aligned" in msg or ("256" in msg and "384" in msg) + """Recognize the legacy vector-shape failure without returning its text.""" + message = str(exc) + return "not aligned" in message or ("256" in message and "384" in message) def _keyword_search(ws, q, limit=20): @@ -246,13 +278,14 @@ def bootstrap(): ).get("name") emb = None try: - from engraphis.backends import embedder_st as _est from engraphis.backends.embedder_deterministic import DeterministicEmbedder e = current_service.engine.embedder d = int(getattr(e, "dim", 0)) semantic = not isinstance(e, DeterministicEmbedder) + # ``LAST_EMBEDDER_ERROR`` may contain a provider exception. The bootstrap + # contract exposes capability metadata only, never the failure text. emb = {"class": type(e).__name__, "dim": d, "semantic": semantic, - "model": settings.embed_model, "error": getattr(_est, "LAST_EMBEDDER_ERROR", "")} + "model": settings.embed_model} except Exception: # noqa: BLE001 pass return { @@ -576,10 +609,9 @@ def llm_activity(workspace: Optional[str] = None, limit: int = 100): return {"workspace": "", "count": 0, "activities": []} try: ws = service()._clean_ws(ws) - except ValidationError as exc: - logger.info("LLM activity request rejected (%s)", type(exc).__name__) - # codeql[py/stack-trace-exposure] ValidationError is safe public input feedback. - raise HTTPException(status_code=400, detail={"error": str(exc)}) from None + except ValidationError: + logger.info("LLM activity request rejected") + raise _invalid_request() from None row = service().store.conn.execute( "SELECT id FROM workspaces WHERE name=?", (ws,) ).fetchone() @@ -836,10 +868,9 @@ def recall(q: str = Query(...), workspace: Optional[str] = None, k: int = 8, mtypes = [mtype] if mtype else None try: out = service().recall(q, workspace=ws, k=k, mtypes=mtypes, reinforce=False) - except ValidationError as exc: - logger.info("dashboard recall request rejected (%s)", type(exc).__name__) - # codeql[py/stack-trace-exposure] ValidationError is safe public input feedback. - raise HTTPException(status_code=400, detail={"error": str(exc)}) from None + except ValidationError: + logger.info("dashboard recall request rejected") + raise _invalid_request() from None except Exception as exc: # noqa: BLE001 if not _is_embedder_mismatch(exc): logger.error("dashboard recall failed (%s)", type(exc).__name__) @@ -866,10 +897,9 @@ def memories(workspace: Optional[str] = None, q: Optional[str] = None, limit: in return {"workspace": "", "count": 0, "memories": []} try: ws = service()._clean_ws(ws) - except ValidationError as exc: - logger.info("dashboard memories request rejected (%s)", type(exc).__name__) - # codeql[py/stack-trace-exposure] ValidationError is safe public input feedback. - raise HTTPException(status_code=400, detail={"error": str(exc)}) from None + except ValidationError: + logger.info("dashboard memories request rejected") + raise _invalid_request() from None conn = _sql.connect("file:%s?mode=ro" % settings.db_path, uri=True) conn.row_factory = _sql.Row try: @@ -922,10 +952,9 @@ def why(q: str = Query(...), workspace: Optional[str] = None, k: int = 5): ws = workspace or _require_ws() try: out = service().why(q, workspace=ws, k=k) - except ValidationError as exc: - logger.info("dashboard why request rejected (%s)", type(exc).__name__) - # codeql[py/stack-trace-exposure] ValidationError is safe public input feedback. - raise HTTPException(status_code=400, detail={"error": str(exc)}) from None + except ValidationError: + logger.info("dashboard why request rejected") + raise _invalid_request() from None except Exception as exc: # noqa: BLE001 if not _is_embedder_mismatch(exc): logger.error("dashboard why failed (%s)", type(exc).__name__) @@ -944,10 +973,9 @@ def timeline(q: str = Query(...), workspace: Optional[str] = None, limit: int = ws = workspace or _default_ws() try: out = service().timeline(q, workspace=ws, limit=limit) - except ValidationError as exc: - logger.info("dashboard timeline request rejected (%s)", type(exc).__name__) - # codeql[py/stack-trace-exposure] ValidationError is safe public input feedback. - raise HTTPException(status_code=400, detail={"error": str(exc)}) from None + except ValidationError: + logger.info("dashboard timeline request rejected") + raise _invalid_request() from None except Exception as exc: # noqa: BLE001 if not _is_embedder_mismatch(exc): logger.error("dashboard timeline failed (%s)", type(exc).__name__) @@ -1448,6 +1476,13 @@ def graph_scene(workspace: Optional[str] = None, level: str = "overview", level.strip().lower() == "complete" ) code_enabled = include_code if code_overlay is None else code_overlay + if level.strip().lower() == "complete" and (node_limit is not None or edge_limit is not None): + # This is route-owned, parameter-only validation. It keeps the precise dashboard + # guidance without ever serializing a service exception. + raise HTTPException(status_code=400, detail={ + "error": "complete scenes do not accept node_limit or edge_limit; " + "use graph filters instead of silently truncating the chart", + }) return _run( service().graph_scene, workspace=ws, level=level, center_id=center_id, system_id=system_id, seeds=_graph_csv(seeds), @@ -1565,11 +1600,43 @@ class _CodeIndexReq(BaseModel): languages: Optional[list] = None +def _http_code_index_path(root_path: str) -> str: + """Resolve an HTTP indexing target beneath the single operator-owned root. + + The HTTP API is a remote trust boundary, unlike direct local MCP and CLI use. + Keep its root independent from the broader engine allow-list and pass only the + checked, canonical path to the service layer. + """ + configured_root = os.environ.get("ENGRAPHIS_HTTP_INDEX_ROOT", "").strip() + if not configured_root: + configured_roots = os.environ.get("ENGRAPHIS_INDEX_ROOTS", "").split(os.pathsep) + configured_root = next((value.strip() for value in configured_roots if value.strip()), "") + if configured_root and not os.path.isabs(configured_root): + raise _HttpCodeIndexConfigurationError("HTTP code index root must be configured absolutely") + + base = os.path.normcase(os.path.realpath(configured_root or os.getcwd())) + candidate = os.path.normcase(os.path.realpath(os.path.join(base, root_path))) + + # Keep a separator on both values so /operator/root-copy cannot pass as a + # child of /operator/root. The root itself remains an allowed target. + base_prefix = base.rstrip(os.sep) + os.sep + candidate_with_sep = candidate.rstrip(os.sep) + os.sep + if not candidate_with_sep.startswith(base_prefix): + raise ValidationError("code index root is outside the HTTP index root") + return candidate_with_sep + + @router.post("/code/index") def code_index(req: _CodeIndexReq): + try: + root_path = _http_code_index_path(req.root_path) + except _HttpCodeIndexConfigurationError: + raise _http_index_configuration_error() from None + except ValidationError: + raise _invalid_request() from None return _run( service().index_repo, workspace=req.workspace, repo=req.repo, - root_path=req.root_path, languages=req.languages, + root_path=root_path, languages=req.languages, ) diff --git a/engraphis/service.py b/engraphis/service.py index 73d542c..c501052 100644 --- a/engraphis/service.py +++ b/engraphis/service.py @@ -1141,7 +1141,8 @@ def _import_one(self, name: str, content: str, *, ws: str, mt: MemoryType, ) return {"file": name, "id": r["id"], "op": r["op"]} except ValidationError as exc: - return {"file": name, "error": str(exc)} + logger.info("uploaded resource import rejected (%s)", type(exc).__name__) + return {"file": name, "error": "resource could not be imported"} def _derive_import_facts(self, content: str, *, ws: str, mt: MemoryType, resource_name: str, resource_kind: str, @@ -1335,8 +1336,9 @@ def import_files(self, *, workspace: str, files: list, memory_type: str = "seman if "no extractable text" in str(exc): skipped += 1 continue + logger.info("uploaded resource extraction failed (%s)", type(exc).__name__) errors += 1 - details.append({"file": name, "error": str(exc)}) + details.append({"file": name, "error": "resource could not be imported"}) continue resource_meta = { **resource.metadata, @@ -1368,7 +1370,9 @@ def import_files(self, *, workspace: str, files: list, memory_type: str = "seman if note: file_warnings.append(note) except (OSError, ValueError) as exc: - file_warnings.append(f"fact derivation failed: {exc}") + logger.info("uploaded resource fact derivation failed (%s)", + type(exc).__name__) + file_warnings.append("fact derivation failed") if file_warnings: warnings.append({"file": name, "warnings": file_warnings}) diff --git a/tests/test_backends_factories.py b/tests/test_backends_factories.py index 76f7077..baee1aa 100644 --- a/tests/test_backends_factories.py +++ b/tests/test_backends_factories.py @@ -1,3 +1,5 @@ +import hashlib + from engraphis.backends.embedder_deterministic import DeterministicEmbedder from engraphis.backends.embedder_st import get_embedder from engraphis.backends.reranker import IdentityReranker, get_reranker @@ -12,6 +14,14 @@ def test_embedder_factory_falls_back_offline(): assert isinstance(get_embedder("definitely-not-a-real-model-xyz", 128), DeterministicEmbedder) +def test_deterministic_embedder_preserves_legacy_feature_hash_mapping(): + """Changing the feature-hash algorithm would invalidate existing local vectors.""" + vectors = DeterministicEmbedder(dim=64).embed(["alpha beta graph", "offline mapping 123"]) + assert hashlib.sha256(vectors.tobytes()).hexdigest() == ( + "c2378cd31c56863b0c65fe7b0634aa62250af35b94853298bfed34fbb71875df" + ) + + def test_vector_index_factory_modes(monkeypatch): """prefer="numpy" always forces the reference index; prefer="auto" returns the best AVAILABLE backend — asserted for both availability branches explicitly diff --git a/tests/test_code_index_route_security.py b/tests/test_code_index_route_security.py new file mode 100644 index 0000000..72adb2e --- /dev/null +++ b/tests/test_code_index_route_security.py @@ -0,0 +1,154 @@ +"""The remote code-index endpoint is confined to one operator-owned root.""" +import os + +import pytest + +pytest.importorskip("fastapi") + +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from engraphis.routes import v2_api + + +class _IndexService: + def __init__(self): + self.calls = [] + + def index_repo(self, **kwargs): + self.calls.append(kwargs) + return {"files_indexed": 0} + + +@pytest.fixture() +def indexed_client(monkeypatch): + service = _IndexService() + v2_api.set_service(service) + app = FastAPI() + app.include_router(v2_api.router) + yield TestClient(app), service + v2_api._service = None + + +def _request(client, root_path): + return client.post("/api/code/index", json={ + "workspace": "acme", "repo": "sample", "root_path": root_path, + }) + + +def _canonical_with_sep(path): + return os.path.normcase(os.path.realpath(path)).rstrip(os.sep) + os.sep + + +def test_code_index_uses_configured_http_root_for_relative_paths( + indexed_client, monkeypatch, tmp_path, +): + client, service = indexed_client + root = tmp_path / "operator-root" + target = root / "project" + target.mkdir(parents=True) + monkeypatch.setenv("ENGRAPHIS_HTTP_INDEX_ROOT", str(root)) + + response = _request(client, "project") + + assert response.status_code == 200 + assert service.calls[0]["root_path"] == _canonical_with_sep(target) + + +def test_code_index_allows_absolute_path_only_inside_http_root(indexed_client, monkeypatch, tmp_path): + client, service = indexed_client + root = tmp_path / "operator-root" + target = root / "project" + target.mkdir(parents=True) + monkeypatch.setenv("ENGRAPHIS_HTTP_INDEX_ROOT", str(root)) + + response = _request(client, str(target)) + + assert response.status_code == 200 + assert service.calls[0]["root_path"] == _canonical_with_sep(target) + + +def test_code_index_rejects_outside_and_prefix_sibling_paths(indexed_client, monkeypatch, tmp_path): + client, service = indexed_client + root = tmp_path / "operator-root" + sibling = tmp_path / "operator-root-copy" + root.mkdir() + sibling.mkdir() + monkeypatch.setenv("ENGRAPHIS_HTTP_INDEX_ROOT", str(root)) + + for path in ("../operator-root-copy", str(sibling)): + response = _request(client, path) + assert response.status_code == 400 + assert response.json() == {"detail": {"error": "invalid request"}} + assert service.calls == [] + + +def test_code_index_rejects_symlink_escape(indexed_client, monkeypatch, tmp_path): + client, service = indexed_client + root = tmp_path / "operator-root" + outside = tmp_path / "outside" + root.mkdir() + outside.mkdir() + link = root / "escape" + try: + link.symlink_to(outside, target_is_directory=True) + except (OSError, NotImplementedError): + pytest.skip("directory symlinks are unavailable in this environment") + monkeypatch.setenv("ENGRAPHIS_HTTP_INDEX_ROOT", str(root)) + + assert _request(client, "escape").status_code == 400 + assert service.calls == [] + + +def test_code_index_uses_first_engine_root_when_http_root_is_unset(indexed_client, monkeypatch, tmp_path): + client, service = indexed_client + first = tmp_path / "first" + second = tmp_path / "second" + target = first / "project" + target.mkdir(parents=True) + second.mkdir() + monkeypatch.delenv("ENGRAPHIS_HTTP_INDEX_ROOT", raising=False) + monkeypatch.setenv("ENGRAPHIS_INDEX_ROOTS", os.pathsep.join((str(first), str(second)))) + + response = _request(client, "project") + + assert response.status_code == 200 + assert service.calls[0]["root_path"] == _canonical_with_sep(target) + + +@pytest.mark.parametrize( + ("http_root", "engine_roots"), + [ + ("relative-root", None), + (None, "relative-root"), + ], +) +def test_code_index_rejects_relative_operator_root_configuration( + indexed_client, monkeypatch, http_root, engine_roots, +): + client, service = indexed_client + if http_root is None: + monkeypatch.delenv("ENGRAPHIS_HTTP_INDEX_ROOT", raising=False) + else: + monkeypatch.setenv("ENGRAPHIS_HTTP_INDEX_ROOT", http_root) + if engine_roots is None: + monkeypatch.delenv("ENGRAPHIS_INDEX_ROOTS", raising=False) + else: + monkeypatch.setenv("ENGRAPHIS_INDEX_ROOTS", engine_roots) + + response = _request(client, "project") + + assert response.status_code == 500 + assert response.json() == {"detail": {"error": "internal server error"}} + assert service.calls == [] + + +def test_code_index_uses_current_directory_without_operator_root_configuration(indexed_client, monkeypatch): + client, service = indexed_client + monkeypatch.delenv("ENGRAPHIS_HTTP_INDEX_ROOT", raising=False) + monkeypatch.delenv("ENGRAPHIS_INDEX_ROOTS", raising=False) + + response = _request(client, ".") + + assert response.status_code == 200 + assert service.calls[0]["root_path"] == _canonical_with_sep(os.getcwd()) diff --git a/tests/test_codeql_sarif_gate.py b/tests/test_codeql_sarif_gate.py index c941161..84f8c1e 100644 --- a/tests/test_codeql_sarif_gate.py +++ b/tests/test_codeql_sarif_gate.py @@ -49,6 +49,32 @@ def test_codeql_gate_reports_and_rejects_findings(tmp_path, capsys) -> None: assert "py/example at engraphis/example.py:12" in captured.err +def test_codeql_gate_rejects_baselined_and_source_suppressed_findings(tmp_path, capsys) -> None: + _write_sarif( + tmp_path, + [ + { + "ruleId": "py/baselined-example", + "baselineState": "unchanged", + "message": {"text": "existing finding"}, + }, + { + "ruleId": "py/source-suppressed-example", + "suppressions": [{"kind": "inSource"}], + "message": {"text": "suppressed finding"}, + }, + ], + ) + + # Count every raw result. Do not inherit pull-request baselining or silently + # ignore a SARIF suppression. + assert main([str(tmp_path)]) == 1 + captured = capsys.readouterr() + assert "CodeQL gate: 2 finding(s)" in captured.err + assert "py/baselined-example" in captured.err + assert "py/source-suppressed-example" in captured.err + + def test_codeql_gate_rejects_missing_sarif(tmp_path, capsys) -> None: assert main([str(tmp_path)]) == 2 assert "no SARIF files found" in capsys.readouterr().err diff --git a/tests/test_dashboard_v2.py b/tests/test_dashboard_v2.py index 9f0bf21..c4a81d2 100644 --- a/tests/test_dashboard_v2.py +++ b/tests/test_dashboard_v2.py @@ -12,7 +12,7 @@ from engraphis.config import settings # noqa: E402 from engraphis.cloud_features import CloudFeatureError # noqa: E402 from engraphis.routes import v2_api # noqa: E402 -from engraphis.service import MemoryService # noqa: E402 +from engraphis.service import MemoryService, ValidationError # noqa: E402 def _client(monkeypatch, tmp_path): @@ -280,6 +280,36 @@ def fail_with(exc): assert internal.value.detail == {"error": "internal server error"} assert secret not in repr(internal.value.detail) + with pytest.raises(HTTPException) as validation: + v2_api._run(fail_with, ValidationError(secret)) + assert validation.value.status_code == 400 + assert validation.value.detail == {"error": "invalid request"} + assert secret not in repr(validation.value.detail) + + with pytest.raises(HTTPException) as downstream: + v2_api._run(fail_with, HTTPException(status_code=418, detail={"error": secret})) + assert downstream.value.status_code == 418 + assert downstream.value.detail == {"error": "request rejected"} + assert secret not in repr(downstream.value.detail) + + with pytest.raises(HTTPException) as invalid_status: + v2_api._run(fail_with, HTTPException(status_code=999, detail={"error": secret})) + assert invalid_status.value.status_code == 500 + assert invalid_status.value.detail == {"error": "internal server error"} + assert secret not in repr(invalid_status.value.detail) + + with pytest.raises(HTTPException) as mismatch: + v2_api._run(fail_with, ValueError(f"{secret}: shapes 256 and 384 are not aligned")) + assert mismatch.value.status_code == 409 + assert mismatch.value.detail["embedder"] is True + assert secret not in repr(mismatch.value.detail) + + with pytest.raises(HTTPException) as ordinary_value_error: + v2_api._run(fail_with, ValueError(secret)) + assert ordinary_value_error.value.status_code == 500 + assert ordinary_value_error.value.detail == {"error": "internal server error"} + assert secret not in repr(ordinary_value_error.value.detail) + with pytest.raises(HTTPException) as managed: v2_api._managed_call(fail_with, CloudFeatureError(secret, status=502)) assert managed.value.status_code == 502 diff --git a/tests/test_engine.py b/tests/test_engine.py index e7ba01c..3a31280 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -1,4 +1,6 @@ +import os import sqlite3 +import tempfile import pytest @@ -504,6 +506,172 @@ def test_index_repo_and_search_code(tmp_path): assert "add" in names +def test_index_repo_allows_selected_root_only_within_approved_local_roots(tmp_path, monkeypatch): + from engraphis.core import engine as engine_module + + allowed = tmp_path / "allowed" + selected_repo = allowed / "chosen-project" + selected_repo.mkdir(parents=True) + (selected_repo / "module.py").write_text("def selected(): pass\n", encoding="utf-8") + outside = tmp_path / "outside" + outside.mkdir() + (outside / "module.py").write_text("def rejected(): pass\n", encoding="utf-8") + monkeypatch.setattr(engine_module, "_approved_local_index_roots", lambda: (str(allowed),)) + + eng = MemoryEngine.create(":memory:") + wid = eng.store.get_or_create_workspace("w") + rid = eng.store.get_or_create_repo(wid, "sample") + + report = eng.index_repo(rid, str(selected_repo), prefer="regex") + assert report["files_indexed"] == 1 + with pytest.raises(ValueError, match="outside approved local roots"): + eng.index_repo(rid, str(outside), prefer="regex") + + +def test_index_repo_rejects_normalized_escape_from_approved_local_root(tmp_path, monkeypatch): + from engraphis.core import engine as engine_module + + allowed = tmp_path / "allowed" + selected_repo = allowed / "selected-project" + selected_repo.mkdir(parents=True) + outside = tmp_path / "outside" + outside.mkdir() + monkeypatch.setattr(engine_module, "_approved_local_index_roots", lambda: (str(allowed),)) + + eng = MemoryEngine.create(":memory:") + wid = eng.store.get_or_create_workspace("w") + rid = eng.store.get_or_create_repo(wid, "sample") + + escaped = selected_repo / ".." / ".." / "outside" + with pytest.raises(ValueError, match="outside approved local roots"): + eng.index_repo(rid, str(escaped), prefer="regex") + + +def test_index_repo_accepts_the_approved_root_itself(tmp_path, monkeypatch): + from engraphis.core import engine as engine_module + + allowed = tmp_path / "allowed" + allowed.mkdir() + (allowed / "module.py").write_text("def selected(): pass\n", encoding="utf-8") + monkeypatch.setattr(engine_module, "_approved_local_index_roots", lambda: (str(allowed),)) + + eng = MemoryEngine.create(":memory:") + wid = eng.store.get_or_create_workspace("w") + rid = eng.store.get_or_create_repo(wid, "sample") + + report = eng.index_repo(rid, str(allowed), prefer="regex") + assert report["files_indexed"] == 1 + + +def test_index_repo_rejects_root_symlink_that_resolves_outside_approved_root(tmp_path, monkeypatch): + from engraphis.core import engine as engine_module + + allowed = tmp_path / "allowed" + allowed.mkdir() + outside = tmp_path / "outside" + outside.mkdir() + link = allowed / "outside-link" + try: + link.symlink_to(outside, target_is_directory=True) + except (OSError, NotImplementedError): + pytest.skip("symlinks not supported in this environment") + monkeypatch.setattr(engine_module, "_approved_local_index_roots", lambda: (str(allowed),)) + + eng = MemoryEngine.create(":memory:") + wid = eng.store.get_or_create_workspace("w") + rid = eng.store.get_or_create_repo(wid, "sample") + + with pytest.raises(ValueError, match="outside approved local roots"): + eng.index_repo(rid, str(link), prefer="regex") + + +def test_index_repo_operator_roots_replace_local_defaults(tmp_path, monkeypatch): + from engraphis.core import engine as engine_module + + first = tmp_path / "first" + second = tmp_path / "second" + allowed_repo = first / "selected" + allowed_repo.mkdir(parents=True) + (allowed_repo / "module.py").write_text("def selected(): pass\n", encoding="utf-8") + default_only = tmp_path / "outside-configured-roots" + default_only.mkdir() + monkeypatch.setenv("ENGRAPHIS_INDEX_ROOTS", os.pathsep.join((str(first), str(second)))) + + assert engine_module._approved_local_index_roots() == ( + os.path.normcase(os.path.realpath(first)), + os.path.normcase(os.path.realpath(second)), + ) + + eng = MemoryEngine.create(":memory:") + wid = eng.store.get_or_create_workspace("w") + rid = eng.store.get_or_create_repo(wid, "sample") + assert eng.index_repo(rid, str(allowed_repo), prefer="regex")["files_indexed"] == 1 + with pytest.raises(ValueError, match="outside approved local roots"): + eng.index_repo(rid, str(default_only), prefer="regex") + + +def test_index_repo_rejects_relative_operator_roots(monkeypatch): + from engraphis.core import engine as engine_module + + monkeypatch.setenv("ENGRAPHIS_INDEX_ROOTS", "relative-root") + with pytest.raises(ValueError, match="ENGRAPHIS_INDEX_ROOTS.*absolute"): + engine_module._approved_local_index_roots() + + +def test_index_repo_preserves_default_roots_without_operator_configuration(monkeypatch): + from engraphis.core import engine as engine_module + + monkeypatch.delenv("ENGRAPHIS_INDEX_ROOTS", raising=False) + monkeypatch.delenv("ENGRAPHIS_HTTP_INDEX_ROOT", raising=False) + expected = tuple(dict.fromkeys(( + os.path.normcase(os.path.realpath(os.getcwd())), + os.path.normcase(os.path.realpath(os.path.expanduser("~"))), + os.path.normcase(os.path.realpath(tempfile.gettempdir())), + ))) + + assert engine_module._approved_local_index_roots() == expected + + +def test_index_repo_rejects_relative_http_operator_root(monkeypatch): + from engraphis.core import engine as engine_module + + monkeypatch.delenv("ENGRAPHIS_INDEX_ROOTS", raising=False) + monkeypatch.setenv("ENGRAPHIS_HTTP_INDEX_ROOT", "relative-http-root") + with pytest.raises(ValueError, match="ENGRAPHIS_HTTP_INDEX_ROOT.*absolute"): + engine_module._approved_local_index_roots() + + +def test_index_repo_http_root_is_an_approved_engine_root(tmp_path, monkeypatch): + from engraphis.core import engine as engine_module + + http_root = tmp_path / "dedicated-http-root" + selected_repo = http_root / "project" + selected_repo.mkdir(parents=True) + (selected_repo / "module.py").write_text("def selected(): pass\n", encoding="utf-8") + monkeypatch.delenv("ENGRAPHIS_INDEX_ROOTS", raising=False) + monkeypatch.setenv("ENGRAPHIS_HTTP_INDEX_ROOT", str(http_root)) + + roots = engine_module._approved_local_index_roots() + assert os.path.normcase(os.path.realpath(http_root)) in roots + + eng = MemoryEngine.create(":memory:") + wid = eng.store.get_or_create_workspace("w") + rid = eng.store.get_or_create_repo(wid, "sample") + assert eng.index_repo(rid, str(selected_repo), prefer="regex")["files_indexed"] == 1 + + +def test_index_repo_deduplicates_canonical_operator_roots(tmp_path, monkeypatch): + from engraphis.core import engine as engine_module + + root = tmp_path / "operator-root" + root.mkdir() + canonical = os.path.normcase(os.path.realpath(root)) + monkeypatch.setenv("ENGRAPHIS_INDEX_ROOTS", os.pathsep.join((str(root), str(root / ".")))) + monkeypatch.setenv("ENGRAPHIS_HTTP_INDEX_ROOT", str(root)) + + assert engine_module._approved_local_index_roots() == (canonical,) + + def test_index_repo_is_idempotent_per_file(tmp_path): eng = MemoryEngine.create(":memory:") wid = eng.store.get_or_create_workspace("w") diff --git a/tests/test_import_error_redaction.py b/tests/test_import_error_redaction.py new file mode 100644 index 0000000..37a3061 --- /dev/null +++ b/tests/test_import_error_redaction.py @@ -0,0 +1,93 @@ +"""Regression coverage for safe import-files error responses. + +Import inputs and extractors can include sensitive local data in exception messages. The +service must retain its stable report contract without reflecting those messages. +""" +from types import SimpleNamespace + +from engraphis.backends import resources +from engraphis.service import MemoryService, ValidationError + + +_REPORT_KEYS = { + "workspace", "scanned", "imported", "skipped", "errors", "derived_facts", + "details", "warnings", +} + + +def _assert_report_shape(report): + assert set(report) == _REPORT_KEYS + assert report["workspace"] == "redaction" + assert report["scanned"] == 1 + assert isinstance(report["details"], list) + assert isinstance(report["warnings"], list) + + +def test_import_files_redacts_validation_error_from_memory_write(monkeypatch): + svc = MemoryService.create(":memory:", graph_extractor="none") + secret = "VALIDATION_IMPORT_SECRET_must_not_escape" + + def reject_write(*_args, **_kwargs): + raise ValidationError(secret) + + monkeypatch.setattr(svc, "remember", reject_write) + report = svc.import_files( + workspace="redaction", files=[{"name": "note.md", "content": "note"}] + ) + + _assert_report_shape(report) + assert report["imported"] == 0 + assert report["skipped"] == 0 + assert report["errors"] == 1 + assert report["derived_facts"] == 0 + assert report["details"] == [{"file": "note.md", "error": "resource could not be imported"}] + assert secret not in repr(report) + + +def test_import_files_redacts_resource_extractor_value_error(monkeypatch): + svc = MemoryService.create(":memory:", graph_extractor="none") + secret = "EXTRACTOR_IMPORT_SECRET_must_not_escape" + + def fail_extract(*_args, **_kwargs): + raise ValueError(secret) + + monkeypatch.setattr( + resources, + "get_resource_extractor", + lambda: SimpleNamespace(extract_bytes=fail_extract), + ) + report = svc.import_files( + workspace="redaction", files=[{"name": "note.md", "content": "note"}] + ) + + _assert_report_shape(report) + assert report["imported"] == 0 + assert report["skipped"] == 0 + assert report["errors"] == 1 + assert report["derived_facts"] == 0 + assert report["details"] == [{"file": "note.md", "error": "resource could not be imported"}] + assert secret not in repr(report) + + +def test_import_files_redacts_derive_facts_value_error(monkeypatch): + svc = MemoryService.create(":memory:", graph_extractor="none") + secret = "DERIVE_FACTS_IMPORT_SECRET_must_not_escape" + + def fail_derivation(*_args, **_kwargs): + raise ValueError(secret) + + monkeypatch.setattr(svc, "_derive_import_facts", fail_derivation) + report = svc.import_files( + workspace="redaction", + files=[{"name": "note.md", "content": "note"}], + derive_facts=True, + ) + + _assert_report_shape(report) + assert report["imported"] == 1 + assert report["skipped"] == 0 + assert report["errors"] == 0 + assert report["derived_facts"] == 0 + assert report["details"] == [] + assert report["warnings"] == [{"file": "note.md", "warnings": ["fact derivation failed"]}] + assert secret not in repr(report) diff --git a/tests/test_protocol_upgrades.py b/tests/test_protocol_upgrades.py index 7b0d9b0..d1af079 100644 --- a/tests/test_protocol_upgrades.py +++ b/tests/test_protocol_upgrades.py @@ -71,15 +71,15 @@ def test_link_infers_layer_for_response_receipt_and_persistence(): def test_receipts_are_content_free_chained_and_tamper_evident(): svc = MemoryService.create(":memory:", graph_extractor="none") - secret = "launch code ORANGE-UNICORN-991" + content_marker = "receipt exclusion marker ORANGE-UNICORN-991" svc.remember( - secret, workspace="w", scope="workspace", + content_marker, workspace="w", scope="workspace", source="alice@example.test", ) svc.recall("ORANGE-UNICORN", workspace="w", reinforce=False) exported = svc.export_receipts(workspace="w") encoded = json.dumps(exported) - assert secret not in encoded + assert content_marker not in encoded assert "alice@example.test" not in encoded assert exported["verification"]["valid"] is True actor = svc.store.conn.execute( diff --git a/tests/test_release_infrastructure.py b/tests/test_release_infrastructure.py index ec15b0d..a7322c4 100644 --- a/tests/test_release_infrastructure.py +++ b/tests/test_release_infrastructure.py @@ -101,6 +101,9 @@ def test_ci_and_release_default_to_read_only_repository_permissions(): def test_codeql_workflow_fails_when_sarif_contains_findings(): workflow = _text(".github/workflows/codeql.yml") + # CodeQL's PR default is diff-informed and omits findings outside the + # patch. The release gate must instead inspect complete raw SARIF. + assert 'CODEQL_ACTION_DIFF_INFORMED_QUERIES: "false"' in workflow assert "id: analyze" in workflow assert "output: codeql-results" in workflow assert (