diff --git a/README.md b/README.md index 0c14d207c..2aaeb0a1e 100644 --- a/README.md +++ b/README.md @@ -270,6 +270,7 @@ Codex users also need `multi_agent = true` under `[features]` in `~/.codex/confi | `ocaml` | OCaml `.ml`/`.mli` AST extraction | `uv tool install "graphifyy[ocaml]"` | | `commonlisp` | Common Lisp `.lisp`/`.cl`/`.lsp`/`.asd` AST extraction | `uv tool install "graphifyy[commonlisp]"` | | `chinese` | Chinese query segmentation (jieba) | `uv tool install "graphifyy[chinese]"` | +| `japanese` | Optional Japanese query segmentation (Janome) | `uv tool install "graphifyy[japanese]"` | | `all` | Everything above | `uv tool install "graphifyy[all]"` | @@ -451,6 +452,8 @@ graphify-out/cost.json # local only # query the graph from the terminal graphify query "show the auth flow" graphify query "what connects DigestAuth to Response?" --graph graphify-out/graph.json +# opt in to content-word segmentation for Japanese queries +graphify query "日本語の検索を改善する" --tokenizer janome_content # expose the graph as an MCP server (for repeated tool-call access) python -m graphify.serve graphify-out/graph.json @@ -466,6 +469,13 @@ python -m graphify.serve graphify-out/graph.json --transport http --host 0.0.0.0 The MCP server gives your assistant structured access: `query_graph`, `get_node`, `get_neighbors`, `shortest_path`, `list_prs`, `get_pr_impact`, `triage_prs`. +The default query tokenizer remains unchanged. For Japanese-language queries, +install the optional `japanese` extra and pass `--tokenizer janome_content` to +the CLI, or set `tokenizer` to `janome_content` in the MCP `query_graph` tool. +This can improve term-level matching for Japanese text, which typically does not +separate words with spaces; it is an explicit opt-in because tokenization can +change retrieval results for a given corpus. + ### Shared HTTP server `--transport stdio` (the default) spawns one local server per developer. `--transport http` serves the same tools over the MCP Streamable HTTP transport, so a single shared process can serve the graph for the whole team — clients point their IDE MCP config at `http://:8080/mcp` instead of running graphify locally. diff --git a/graphify/__main__.py b/graphify/__main__.py index 155501a98..84c9b1f72 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -557,6 +557,7 @@ def _run_cli() -> None: print(" --context C explicit edge-context filter (repeatable)") print(" --budget N cap output at N tokens (default 2000)") print(" --graph path to graph.json (default graphify-out/graph.json)") + print(" --tokenizer NAME query tokenizer: baseline (default) or janome_content") print(" affected \"X\" reverse traversal to find nodes impacted by X") print(" --relation R edge relation to traverse in reverse (repeatable)") print(" --depth N reverse traversal depth (default 2)") diff --git a/graphify/cli.py b/graphify/cli.py index 02e55b944..b438fa420 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -969,7 +969,7 @@ def dispatch_command(cmd: str) -> None: sys.exit(1) elif cmd == "query": if len(sys.argv) < 3: - print("Usage: graphify query \"\" [--dfs] [--context C] [--budget N] [--graph path]", file=sys.stderr) + print("Usage: graphify query \"\" [--dfs] [--context C] [--budget N] [--graph path] [--tokenizer NAME]", file=sys.stderr) sys.exit(1) from graphify.serve import _query_graph_text from graphify.security import sanitize_label @@ -981,6 +981,7 @@ def dispatch_command(cmd: str) -> None: budget = 2000 graph_path = _default_graph_path() context_filters: list[str] = [] + tokenizer = "baseline" args = sys.argv[3:] i = 0 while i < len(args): @@ -1004,6 +1005,12 @@ def dispatch_command(cmd: str) -> None: elif args[i].startswith("--context="): context_filters.append(args[i].split("=", 1)[1]) i += 1 + elif args[i] == "--tokenizer" and i + 1 < len(args): + tokenizer = args[i + 1] + i += 2 + elif args[i].startswith("--tokenizer="): + tokenizer = args[i].split("=", 1)[1] + i += 1 elif args[i] == "--graph" and i + 1 < len(args): graph_path = args[i + 1] i += 2 @@ -1067,15 +1074,20 @@ def dispatch_command(cmd: str) -> None: import time as _time _t0 = _time.perf_counter() _mode = "dfs" if use_dfs else "bfs" - _result = _query_graph_text( - G, - question, - mode=_mode, - depth=2, - token_budget=budget, - context_filters=context_filters, - graph_path=str(gp), - ) + try: + _result = _query_graph_text( + G, + question, + mode=_mode, + depth=2, + token_budget=budget, + context_filters=context_filters, + graph_path=str(gp), + tokenizer=tokenizer, + ) + except (ImportError, ValueError) as exc: + print(f"error: {exc}", file=sys.stderr) + sys.exit(1) querylog.log_query( kind="query", question=question, diff --git a/graphify/serve.py b/graphify/serve.py index 4cf6d8396..4fb64efa4 100644 --- a/graphify/serve.py +++ b/graphify/serve.py @@ -21,6 +21,18 @@ except ImportError: _jieba = None +try: + from janome.tokenizer import Tokenizer as _JanomeTokenizer # type: ignore[import-untyped] +except ImportError: + _JanomeTokenizer = None + + +_QUERY_TOKENIZERS = ("baseline", "janome_content") +_JANOME_CONTENT_PARTS = frozenset({"名詞", "動詞", "形容詞", "副詞"}) +_JANOME_TOKENIZER_INSTANCE = None +_JANOME_TOKENIZER_FACTORY = None +_JANOME_TOKENIZER_LOCK = threading.Lock() + def _load_graph(graph_path: str) -> nx.Graph: try: @@ -198,6 +210,36 @@ def _segment_chinese(text: str) -> list[str]: return segments +def _get_janome_tokenizer(): + """Return the lazily initialized Janome tokenizer singleton.""" + global _JANOME_TOKENIZER_FACTORY, _JANOME_TOKENIZER_INSTANCE + + tokenizer_factory = _JanomeTokenizer + if tokenizer_factory is None: + raise ImportError( + 'The "janome_content" tokenizer requires the optional Japanese extra. ' + 'Install it with: uv tool install "graphifyy[japanese]"' + ) + + if _JANOME_TOKENIZER_INSTANCE is None or _JANOME_TOKENIZER_FACTORY is not tokenizer_factory: + with _JANOME_TOKENIZER_LOCK: + if _JANOME_TOKENIZER_INSTANCE is None or _JANOME_TOKENIZER_FACTORY is not tokenizer_factory: + _JANOME_TOKENIZER_INSTANCE = tokenizer_factory() + _JANOME_TOKENIZER_FACTORY = tokenizer_factory + return _JANOME_TOKENIZER_INSTANCE + + +def _segment_japanese_content(text: str) -> list[str]: + """Return Janome surface forms for Japanese content words.""" + terms: list[str] = [] + for token in _get_janome_tokenizer().tokenize(text): + part = str(getattr(token, "part_of_speech", "")).split(",", 1)[0] + surface = str(getattr(token, "surface", "")).strip().lower() + if part in _JANOME_CONTENT_PARTS and surface: + terms.append(surface) + return terms + + def _is_searchable(term: str) -> bool: """True if term is Chinese, non-English, or an English word longer than 2 chars.""" if all("a" <= ch <= "z" for ch in term): @@ -259,25 +301,36 @@ def _is_searchable(term: str) -> bool: }) -def _query_terms(question: str) -> list[str]: - """Split a query into searchable terms, segmenting Chinese text, then drop - question/filler words (`_QUERY_STOPWORDS`, English plus common German/ - Romance-language fillers) so content words drive seeding. Falls back to the - unfiltered terms if the query is all stopwords, so a question like "how does - it work" or "wie funktioniert das" still seeds on something.""" - terms: list[str] = [] - for raw in question.split(): - if _has_chinese(raw): - for seg in _segment_chinese(raw.lower().strip()): - seg = seg.strip() - if seg and _is_searchable(seg): - terms.append(seg) - else: - # Strip punctuation without touching Unicode characters (avoid NFKD mangling non-Latin scripts) - for tok in re.findall(r"\w+", raw.lower()): - if _is_searchable(tok): - terms.append(tok) - content = [t for t in terms if t not in _QUERY_STOPWORDS] +def _query_terms(question: str, *, tokenizer: str = "baseline") -> list[str]: + """Split a query into searchable terms, then drop question/filler words. + + ``baseline`` preserves the existing whitespace/Chinese segmentation behavior. + ``janome_content`` is an explicit opt-in for Japanese content-word + segmentation and requires the optional ``japanese`` extra. + + The all-stopword fallback keeps a question like "how does it work" or + "wie funktioniert das" searchable. + """ + if tokenizer == "janome_content": + terms = _segment_japanese_content(question) + elif tokenizer == "baseline": + terms = [] + for raw in question.split(): + if _has_chinese(raw): + for seg in _segment_chinese(raw.lower().strip()): + seg = seg.strip() + if seg and _is_searchable(seg): + terms.append(seg) + else: + # Strip punctuation without touching Unicode characters (avoid NFKD mangling non-Latin scripts) + for tok in re.findall(r"\w+", raw.lower()): + if _is_searchable(tok): + terms.append(tok) + else: + supported = ", ".join(_QUERY_TOKENIZERS) + raise ValueError(f"Unsupported query tokenizer {tokenizer!r}; choose from {supported}") + + content = [t for t in terms if t.lower() not in _QUERY_STOPWORDS] return content or terms @@ -1193,8 +1246,9 @@ def _query_graph_text( token_budget: int = 2000, context_filters: list[str] | None = None, graph_path: str | None = None, + tokenizer: str = "baseline", ) -> str: - terms = _query_terms(question) + terms = _query_terms(question, tokenizer=tokenizer) # One graph scoring pass produces both the combined ranking (used to drive # the gap-based seed selection below) and the per-token singleton winners # (used by _pick_seeds' per-term guarantee). Previously this was T+1 passes @@ -1589,6 +1643,12 @@ async def list_tools() -> list[types.Tool]: "description": "bfs=broad context, dfs=trace a specific path"}, "depth": {"type": "integer", "default": 3, "description": "Traversal depth (1-6)"}, "token_budget": {"type": "integer", "default": 2000, "description": "Max output tokens"}, + "tokenizer": { + "type": "string", + "enum": list(_QUERY_TOKENIZERS), + "default": "baseline", + "description": "Query tokenizer; janome_content is an explicit Japanese opt-in", + }, "context_filter": { "type": "array", "items": {"type": "string"}, @@ -1734,6 +1794,7 @@ def _tool_query_graph(arguments: dict) -> str: mode = arguments.get("mode", "bfs") depth = min(int(arguments.get("depth", 3)), 6) budget = int(arguments.get("token_budget", 2000)) + tokenizer = arguments.get("tokenizer", "baseline") context_filter = arguments.get("context_filter") _t0 = _time.perf_counter() result = _query_graph_text( @@ -1744,6 +1805,7 @@ def _tool_query_graph(arguments: dict) -> str: token_budget=budget, context_filters=context_filter, graph_path=str(active_graph_path), + tokenizer=tokenizer, ) querylog.log_query( kind="mcp_query", diff --git a/pyproject.toml b/pyproject.toml index 15ea9dd57..6582734dc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,6 +74,7 @@ anthropic = ["anthropic"] gemini = ["openai", "tiktoken"] openai = ["openai", "tiktoken"] chinese = ["jieba"] +japanese = ["Janome"] sql = ["tree-sitter-sql"] # extract_pascal() uses tree-sitter-pascal for AST-quality extraction (more # accurate calls/inherits edges) and falls back to a regex extractor when it is @@ -91,7 +92,7 @@ ocaml = ["tree-sitter-ocaml"] # tree-sitter-commonlisp ships prebuilt abi3 wheels for every platform; optional # because Common Lisp is a niche corpus language. commonlisp = ["tree-sitter-commonlisp"] -all = ["mcp>=1,<3", "starlette>=1.3.1,<2", "neo4j", "falkordb", "pypdf>=6.12.0", "markdownify", "watchdog", "graspologic; python_version < '3.13'", "python-docx", "openpyxl", "faster-whisper; python_version >= '3.11'", "yt-dlp>=2026.6.9", "matplotlib", "numpy>=2.0; python_version >= '3.13'", "openai", "tiktoken", "boto3", "anthropic", "tree-sitter-sql", "jieba", "tree-sitter-dm", "tree-sitter-hcl", "tree-sitter-pascal", "tree-sitter-ocaml", "tree-sitter-commonlisp"] +all = ["mcp>=1,<3", "starlette>=1.3.1,<2", "neo4j", "falkordb", "pypdf>=6.12.0", "markdownify", "watchdog", "graspologic; python_version < '3.13'", "python-docx", "openpyxl", "faster-whisper; python_version >= '3.11'", "yt-dlp>=2026.6.9", "matplotlib", "numpy>=2.0; python_version >= '3.13'", "openai", "tiktoken", "boto3", "anthropic", "tree-sitter-sql", "jieba", "Janome", "tree-sitter-dm", "tree-sitter-hcl", "tree-sitter-pascal", "tree-sitter-ocaml", "tree-sitter-commonlisp"] [project.scripts] graphify = "graphify.__main__:main" diff --git a/tests/test_cli_export.py b/tests/test_cli_export.py index 6a3ee59e6..050342c70 100644 --- a/tests/test_cli_export.py +++ b/tests/test_cli_export.py @@ -188,6 +188,19 @@ def test_query_budget_flag(tmp_path): assert r.returncode == 0, r.stderr +def test_query_baseline_tokenizer_flag(tmp_path): + _make_graph(tmp_path) + r = _run(["query", "test", "--tokenizer", "baseline"], tmp_path) + assert r.returncode == 0, r.stderr + + +def test_query_unknown_tokenizer_fails(tmp_path): + _make_graph(tmp_path) + r = _run(["query", "test", "--tokenizer", "unknown"], tmp_path) + assert r.returncode != 0 + assert "Unsupported query tokenizer" in r.stderr + + def test_query_missing_graph_fails(tmp_path): r = _run(["query", "anything"], tmp_path) assert r.returncode != 0 diff --git a/tests/test_serve.py b/tests/test_serve.py index 85f77a59a..596d89878 100644 --- a/tests/test_serve.py +++ b/tests/test_serve.py @@ -1170,6 +1170,59 @@ def test_query_terms_chinese_no_jieba_fallback(monkeypatch): assert len(terms) == 4 +def test_query_terms_janome_content_keeps_content_parts(monkeypatch): + """The Japanese opt-in keeps Janome content-word surface forms.""" + import graphify.serve as serve_mod + + class FakeToken: + def __init__(self, surface, part_of_speech): + self.surface = surface + self.part_of_speech = part_of_speech + + class FakeTokenizer: + def tokenize(self, text): + assert text == "日本語の検索を改善する。" + return [ + FakeToken("日本語", "名詞,一般,*,*"), + FakeToken("の", "助詞,連体化,*,*"), + FakeToken("検索", "名詞,サ変接続,*,*"), + FakeToken("改善", "名詞,サ変接続,*,*"), + FakeToken("する", "動詞,自立,*,*"), + FakeToken("。", "記号,句点,*,*"), + ] + + monkeypatch.setattr(serve_mod, "_JanomeTokenizer", FakeTokenizer) + assert _query_terms("日本語の検索を改善する。", tokenizer="janome_content") == [ + "日本語", + "検索", + "改善", + "する", + ] + + +def test_query_terms_janome_content_real_library(): + pytest.importorskip("janome") + terms = _query_terms("日本語の検索を改善する", tokenizer="janome_content") + assert "日本語" in terms + assert "検索" in terms + assert "改善" in terms + assert "の" not in terms + assert "を" not in terms + + +def test_query_terms_janome_content_requires_optional_extra(monkeypatch): + import graphify.serve as serve_mod + + monkeypatch.setattr(serve_mod, "_JanomeTokenizer", None) + with pytest.raises(ImportError, match=r"graphifyy\[japanese\]"): + _query_terms("日本語の検索", tokenizer="janome_content") + + +def test_query_terms_rejects_unknown_tokenizer(): + with pytest.raises(ValueError, match="Unsupported query tokenizer"): + _query_terms("query", tokenizer="unknown") + + def test_score_nodes_chinese_substring_match(): """Searching for '路由' should match a node with label containing '路由'.""" G = nx.Graph() diff --git a/uv.lock b/uv.lock index 881314a5b..02b38a993 100644 --- a/uv.lock +++ b/uv.lock @@ -1133,6 +1133,7 @@ all = [ { name = "falkordb" }, { name = "faster-whisper", marker = "python_full_version >= '3.11'" }, { name = "graspologic", marker = "python_full_version < '3.13'" }, + { name = "janome" }, { name = "jieba" }, { name = "markdownify" }, { name = "matplotlib" }, @@ -1179,6 +1180,9 @@ gemini = [ google = [ { name = "openpyxl" }, ] +japanese = [ + { name = "janome" }, +] kimi = [ { name = "openai" }, { name = "tiktoken" }, @@ -1268,6 +1272,8 @@ requires-dist = [ { name = "faster-whisper", marker = "python_full_version >= '3.11' and extra == 'video'" }, { name = "graspologic", marker = "python_full_version < '3.13' and extra == 'all'" }, { name = "graspologic", marker = "python_full_version < '3.13' and extra == 'leiden'" }, + { name = "janome", marker = "extra == 'all'" }, + { name = "janome", marker = "extra == 'japanese'" }, { name = "jieba", marker = "extra == 'all'" }, { name = "jieba", marker = "extra == 'chinese'" }, { name = "markdownify", marker = "extra == 'all'" }, @@ -1345,7 +1351,7 @@ requires-dist = [ { name = "yt-dlp", marker = "extra == 'all'", specifier = ">=2026.6.9" }, { name = "yt-dlp", marker = "extra == 'video'", specifier = ">=2026.6.9" }, ] -provides-extras = ["mcp", "neo4j", "falkordb", "pdf", "watch", "svg", "leiden", "office", "google", "postgres", "video", "kimi", "ollama", "bedrock", "anthropic", "gemini", "openai", "chinese", "sql", "pascal", "dm", "terraform", "ocaml", "commonlisp", "all"] +provides-extras = ["mcp", "neo4j", "falkordb", "pdf", "watch", "svg", "leiden", "office", "google", "postgres", "video", "kimi", "ollama", "bedrock", "anthropic", "gemini", "openai", "chinese", "japanese", "sql", "pascal", "dm", "terraform", "ocaml", "commonlisp", "all"] [package.metadata.requires-dev] dev = [ @@ -1590,6 +1596,15 @@ version = "0.42.1" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/c6/cb/18eeb235f833b726522d7ebed54f2278ce28ba9438e3135ab0278d9792a2/jieba-0.42.1.tar.gz", hash = "sha256:055ca12f62674fafed09427f176506079bc135638a14e23e25be909131928db2", size = 19214172, upload-time = "2020-01-20T14:27:23.5Z" } +[[package]] +name = "janome" +version = "0.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/08/4e/dc2b1a89a4ffafbf9bf49c8e11f69e28ba8b3ef81d11afe6e9f96caee6cc/Janome-0.5.0.tar.gz", hash = "sha256:ce4a3ed7a4635c2f80139639327d5b1e0381858ad74a3c4a61e8cc83f820400e", size = 18829020, upload-time = "2023-07-01T10:53:09.161Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/73/7d/70f4069f4bbf0fca023e82a1fbbade6f5216365d4fe259fee1950723eca5/Janome-0.5.0-py2.py3-none-any.whl", hash = "sha256:d098670394a77881ce2f6b7d696c0ea5ff74c0c8cf74a8a882159ec82c0e6dc7", size = 19654103, upload-time = "2023-07-01T10:52:58.572Z" }, +] + [[package]] name = "jiter" version = "0.14.0"