Add opt-in Japanese query tokenization

README.md

@@ -270,6 +270,7 @@ Codex users also need `multi_agent = true` under `[features]` in `~/.codex/confi
 | `ocaml` | OCaml `.ml`/`.mli` AST extraction | `uv tool install "graphifyy[ocaml]"` |
 | `commonlisp` | Common Lisp `.lisp`/`.cl`/`.lsp`/`.asd` AST extraction | `uv tool install "graphifyy[commonlisp]"` |
 | `chinese` | Chinese query segmentation (jieba) | `uv tool install "graphifyy[chinese]"` |
+| `japanese` | Optional Japanese query segmentation (Janome) | `uv tool install "graphifyy[japanese]"` |
 | `all` | Everything above | `uv tool install "graphifyy[all]"` |
 
 </details>
@@ -451,6 +452,8 @@ graphify-out/cost.json        # local only
 # query the graph from the terminal
 graphify query "show the auth flow"
 graphify query "what connects DigestAuth to Response?" --graph graphify-out/graph.json
+# opt in to content-word segmentation for Japanese queries
+graphify query "日本語の検索を改善する" --tokenizer janome_content
 
 # expose the graph as an MCP server (for repeated tool-call access)
 python -m graphify.serve graphify-out/graph.json
@@ -466,6 +469,13 @@ python -m graphify.serve graphify-out/graph.json --transport http --host 0.0.0.0
 
 The MCP server gives your assistant structured access: `query_graph`, `get_node`, `get_neighbors`, `shortest_path`, `list_prs`, `get_pr_impact`, `triage_prs`.
 
+The default query tokenizer remains unchanged. For Japanese-language queries,
+install the optional `japanese` extra and pass `--tokenizer janome_content` to
+the CLI, or set `tokenizer` to `janome_content` in the MCP `query_graph` tool.
+This can improve term-level matching for Japanese text, which typically does not
+separate words with spaces; it is an explicit opt-in because tokenization can
+change retrieval results for a given corpus.
+
 ### Shared HTTP server
 
 `--transport stdio` (the default) spawns one local server per developer. `--transport http` serves the same tools over the MCP Streamable HTTP transport, so a single shared process can serve the graph for the whole team — clients point their IDE MCP config at `http://<host>:8080/mcp` instead of running graphify locally.

graphify/__main__.py

@@ -557,6 +557,7 @@ def _run_cli() -> None:
         print("    --context C             explicit edge-context filter (repeatable)")
         print("    --budget N              cap output at N tokens (default 2000)")
         print("    --graph <path>          path to graph.json (default graphify-out/graph.json)")
+        print("    --tokenizer NAME        query tokenizer: baseline (default) or janome_content")
         print("  affected \"X\"             reverse traversal to find nodes impacted by X")
         print("    --relation R            edge relation to traverse in reverse (repeatable)")
         print("    --depth N               reverse traversal depth (default 2)")

graphify/cli.py

@@ -969,7 +969,7 @@ def dispatch_command(cmd: str) -> None:
             sys.exit(1)
     elif cmd == "query":
         if len(sys.argv) < 3:
-            print("Usage: graphify query \"<question>\" [--dfs] [--context C] [--budget N] [--graph path]", file=sys.stderr)
+            print("Usage: graphify query \"<question>\" [--dfs] [--context C] [--budget N] [--graph path] [--tokenizer NAME]", file=sys.stderr)
             sys.exit(1)
         from graphify.serve import _query_graph_text
         from graphify.security import sanitize_label
@@ -981,6 +981,7 @@ def dispatch_command(cmd: str) -> None:
         budget = 2000
         graph_path = _default_graph_path()
         context_filters: list[str] = []
+        tokenizer = "baseline"
         args = sys.argv[3:]
         i = 0
         while i < len(args):
@@ -1004,6 +1005,12 @@ def dispatch_command(cmd: str) -> None:
             elif args[i].startswith("--context="):
                 context_filters.append(args[i].split("=", 1)[1])
                 i += 1
+            elif args[i] == "--tokenizer" and i + 1 < len(args):
+                tokenizer = args[i + 1]
+                i += 2
+            elif args[i].startswith("--tokenizer="):
+                tokenizer = args[i].split("=", 1)[1]
+                i += 1
             elif args[i] == "--graph" and i + 1 < len(args):
                 graph_path = args[i + 1]
                 i += 2
@@ -1067,15 +1074,20 @@ def dispatch_command(cmd: str) -> None:
         import time as _time
         _t0 = _time.perf_counter()
         _mode = "dfs" if use_dfs else "bfs"
-        _result = _query_graph_text(
-            G,
-            question,
-            mode=_mode,
-            depth=2,
-            token_budget=budget,
-            context_filters=context_filters,
-            graph_path=str(gp),
-        )
+        try:
+            _result = _query_graph_text(
+                G,
+                question,
+                mode=_mode,
+                depth=2,
+                token_budget=budget,
+                context_filters=context_filters,
+                graph_path=str(gp),
+                tokenizer=tokenizer,
+            )
+        except (ImportError, ValueError) as exc:
+            print(f"error: {exc}", file=sys.stderr)
+            sys.exit(1)
         querylog.log_query(
             kind="query",
             question=question,

graphify/serve.py

@@ -21,6 +21,18 @@
 except ImportError:
     _jieba = None
 
+try:
+    from janome.tokenizer import Tokenizer as _JanomeTokenizer  # type: ignore[import-untyped]
+except ImportError:
+    _JanomeTokenizer = None
+
+
+_QUERY_TOKENIZERS = ("baseline", "janome_content")
+_JANOME_CONTENT_PARTS = frozenset({"名詞", "動詞", "形容詞", "副詞"})
+_JANOME_TOKENIZER_INSTANCE = None
+_JANOME_TOKENIZER_FACTORY = None
+_JANOME_TOKENIZER_LOCK = threading.Lock()
+
 
 def _load_graph(graph_path: str) -> nx.Graph:
     try:
@@ -198,6 +210,36 @@ def _segment_chinese(text: str) -> list[str]:
     return segments
 
 
+def _get_janome_tokenizer():
+    """Return the lazily initialized Janome tokenizer singleton."""
+    global _JANOME_TOKENIZER_FACTORY, _JANOME_TOKENIZER_INSTANCE
+
+    tokenizer_factory = _JanomeTokenizer
+    if tokenizer_factory is None:
+        raise ImportError(
+            'The "janome_content" tokenizer requires the optional Japanese extra. '
+            'Install it with: uv tool install "graphifyy[japanese]"'
+        )
+
+    if _JANOME_TOKENIZER_INSTANCE is None or _JANOME_TOKENIZER_FACTORY is not tokenizer_factory:
+        with _JANOME_TOKENIZER_LOCK:
+            if _JANOME_TOKENIZER_INSTANCE is None or _JANOME_TOKENIZER_FACTORY is not tokenizer_factory:
+                _JANOME_TOKENIZER_INSTANCE = tokenizer_factory()
+                _JANOME_TOKENIZER_FACTORY = tokenizer_factory
+    return _JANOME_TOKENIZER_INSTANCE
+
+
+def _segment_japanese_content(text: str) -> list[str]:
+    """Return Janome surface forms for Japanese content words."""
+    terms: list[str] = []
+    for token in _get_janome_tokenizer().tokenize(text):
+        part = str(getattr(token, "part_of_speech", "")).split(",", 1)[0]
+        surface = str(getattr(token, "surface", "")).strip().lower()
+        if part in _JANOME_CONTENT_PARTS and surface:
+            terms.append(surface)
+    return terms
+
+
 def _is_searchable(term: str) -> bool:
     """True if term is Chinese, non-English, or an English word longer than 2 chars."""
     if all("a" <= ch <= "z" for ch in term):
@@ -259,25 +301,36 @@ def _is_searchable(term: str) -> bool:
 })
 
 
-def _query_terms(question: str) -> list[str]:
-    """Split a query into searchable terms, segmenting Chinese text, then drop
-    question/filler words (`_QUERY_STOPWORDS`, English plus common German/
-    Romance-language fillers) so content words drive seeding. Falls back to the
-    unfiltered terms if the query is all stopwords, so a question like "how does
-    it work" or "wie funktioniert das" still seeds on something."""
-    terms: list[str] = []
-    for raw in question.split():
-        if _has_chinese(raw):
-            for seg in _segment_chinese(raw.lower().strip()):
-                seg = seg.strip()
-                if seg and _is_searchable(seg):
-                    terms.append(seg)
-        else:
-            # Strip punctuation without touching Unicode characters (avoid NFKD mangling non-Latin scripts)
-            for tok in re.findall(r"\w+", raw.lower()):
-                if _is_searchable(tok):
-                    terms.append(tok)
-    content = [t for t in terms if t not in _QUERY_STOPWORDS]
+def _query_terms(question: str, *, tokenizer: str = "baseline") -> list[str]:
+    """Split a query into searchable terms, then drop question/filler words.
+
+    ``baseline`` preserves the existing whitespace/Chinese segmentation behavior.
+    ``janome_content`` is an explicit opt-in for Japanese content-word
+    segmentation and requires the optional ``japanese`` extra.
+
+    The all-stopword fallback keeps a question like "how does it work" or
+    "wie funktioniert das" searchable.
+    """
+    if tokenizer == "janome_content":
+        terms = _segment_japanese_content(question)
+    elif tokenizer == "baseline":
+        terms = []
+        for raw in question.split():
+            if _has_chinese(raw):
+                for seg in _segment_chinese(raw.lower().strip()):
+                    seg = seg.strip()
+                    if seg and _is_searchable(seg):
+                        terms.append(seg)
+            else:
+                # Strip punctuation without touching Unicode characters (avoid NFKD mangling non-Latin scripts)
+                for tok in re.findall(r"\w+", raw.lower()):
+                    if _is_searchable(tok):
+                        terms.append(tok)
+    else:
+        supported = ", ".join(_QUERY_TOKENIZERS)
+        raise ValueError(f"Unsupported query tokenizer {tokenizer!r}; choose from {supported}")
+
+    content = [t for t in terms if t.lower() not in _QUERY_STOPWORDS]
     return content or terms
 
 
@@ -1193,8 +1246,9 @@ def _query_graph_text(
     token_budget: int = 2000,
     context_filters: list[str] | None = None,
     graph_path: str | None = None,
+    tokenizer: str = "baseline",
 ) -> str:
-    terms = _query_terms(question)
+    terms = _query_terms(question, tokenizer=tokenizer)
     # One graph scoring pass produces both the combined ranking (used to drive
     # the gap-based seed selection below) and the per-token singleton winners
     # (used by _pick_seeds' per-term guarantee). Previously this was T+1 passes
@@ -1589,6 +1643,12 @@ async def list_tools() -> list[types.Tool]:
                                  "description": "bfs=broad context, dfs=trace a specific path"},
                         "depth": {"type": "integer", "default": 3, "description": "Traversal depth (1-6)"},
                         "token_budget": {"type": "integer", "default": 2000, "description": "Max output tokens"},
+                        "tokenizer": {
+                            "type": "string",
+                            "enum": list(_QUERY_TOKENIZERS),
+                            "default": "baseline",
+                            "description": "Query tokenizer; janome_content is an explicit Japanese opt-in",
+                        },
                         "context_filter": {
                             "type": "array",
                             "items": {"type": "string"},
@@ -1734,6 +1794,7 @@ def _tool_query_graph(arguments: dict) -> str:
         mode = arguments.get("mode", "bfs")
         depth = min(int(arguments.get("depth", 3)), 6)
         budget = int(arguments.get("token_budget", 2000))
+        tokenizer = arguments.get("tokenizer", "baseline")
         context_filter = arguments.get("context_filter")
         _t0 = _time.perf_counter()
         result = _query_graph_text(
@@ -1744,6 +1805,7 @@ def _tool_query_graph(arguments: dict) -> str:
             token_budget=budget,
             context_filters=context_filter,
             graph_path=str(active_graph_path),
+            tokenizer=tokenizer,
         )
         querylog.log_query(
             kind="mcp_query",

pyproject.toml

@@ -74,6 +74,7 @@ anthropic = ["anthropic"]
 gemini = ["openai", "tiktoken"]
 openai = ["openai", "tiktoken"]
 chinese = ["jieba"]
+japanese = ["Janome"]
 sql = ["tree-sitter-sql"]
 # extract_pascal() uses tree-sitter-pascal for AST-quality extraction (more
 # accurate calls/inherits edges) and falls back to a regex extractor when it is
@@ -91,7 +92,7 @@ ocaml = ["tree-sitter-ocaml"]
 # tree-sitter-commonlisp ships prebuilt abi3 wheels for every platform; optional
 # because Common Lisp is a niche corpus language.
 commonlisp = ["tree-sitter-commonlisp"]
-all = ["mcp>=1,<3", "starlette>=1.3.1,<2", "neo4j", "falkordb", "pypdf>=6.12.0", "markdownify", "watchdog", "graspologic; python_version < '3.13'", "python-docx", "openpyxl", "faster-whisper; python_version >= '3.11'", "yt-dlp>=2026.6.9", "matplotlib", "numpy>=2.0; python_version >= '3.13'", "openai", "tiktoken", "boto3", "anthropic", "tree-sitter-sql", "jieba", "tree-sitter-dm", "tree-sitter-hcl", "tree-sitter-pascal", "tree-sitter-ocaml", "tree-sitter-commonlisp"]
+all = ["mcp>=1,<3", "starlette>=1.3.1,<2", "neo4j", "falkordb", "pypdf>=6.12.0", "markdownify", "watchdog", "graspologic; python_version < '3.13'", "python-docx", "openpyxl", "faster-whisper; python_version >= '3.11'", "yt-dlp>=2026.6.9", "matplotlib", "numpy>=2.0; python_version >= '3.13'", "openai", "tiktoken", "boto3", "anthropic", "tree-sitter-sql", "jieba", "Janome", "tree-sitter-dm", "tree-sitter-hcl", "tree-sitter-pascal", "tree-sitter-ocaml", "tree-sitter-commonlisp"]
 
 [project.scripts]
 graphify = "graphify.__main__:main"

tests/test_cli_export.py

@@ -188,6 +188,19 @@ def test_query_budget_flag(tmp_path):
     assert r.returncode == 0, r.stderr
 
 
+def test_query_baseline_tokenizer_flag(tmp_path):
+    _make_graph(tmp_path)
+    r = _run(["query", "test", "--tokenizer", "baseline"], tmp_path)
+    assert r.returncode == 0, r.stderr
+
+
+def test_query_unknown_tokenizer_fails(tmp_path):
+    _make_graph(tmp_path)
+    r = _run(["query", "test", "--tokenizer", "unknown"], tmp_path)
+    assert r.returncode != 0
+    assert "Unsupported query tokenizer" in r.stderr
+
+
 def test_query_missing_graph_fails(tmp_path):
     r = _run(["query", "anything"], tmp_path)
     assert r.returncode != 0

tests/test_serve.py

@@ -1170,6 +1170,59 @@ def test_query_terms_chinese_no_jieba_fallback(monkeypatch):
     assert len(terms) == 4
 
 
+def test_query_terms_janome_content_keeps_content_parts(monkeypatch):
+    """The Japanese opt-in keeps Janome content-word surface forms."""
+    import graphify.serve as serve_mod
+
+    class FakeToken:
+        def __init__(self, surface, part_of_speech):
+            self.surface = surface
+            self.part_of_speech = part_of_speech
+
+    class FakeTokenizer:
+        def tokenize(self, text):
+            assert text == "日本語の検索を改善する。"
+            return [
+                FakeToken("日本語", "名詞,一般,*,*"),
+                FakeToken("の", "助詞,連体化,*,*"),
+                FakeToken("検索", "名詞,サ変接続,*,*"),
+                FakeToken("改善", "名詞,サ変接続,*,*"),
+                FakeToken("する", "動詞,自立,*,*"),
+                FakeToken("。", "記号,句点,*,*"),
+            ]
+
+    monkeypatch.setattr(serve_mod, "_JanomeTokenizer", FakeTokenizer)
+    assert _query_terms("日本語の検索を改善する。", tokenizer="janome_content") == [
+        "日本語",
+        "検索",
+        "改善",
+        "する",
+    ]
+
+
+def test_query_terms_janome_content_real_library():
+    pytest.importorskip("janome")
+    terms = _query_terms("日本語の検索を改善する", tokenizer="janome_content")
+    assert "日本語" in terms
+    assert "検索" in terms
+    assert "改善" in terms
+    assert "の" not in terms
+    assert "を" not in terms
+
+
+def test_query_terms_janome_content_requires_optional_extra(monkeypatch):
+    import graphify.serve as serve_mod
+
+    monkeypatch.setattr(serve_mod, "_JanomeTokenizer", None)
+    with pytest.raises(ImportError, match=r"graphifyy\[japanese\]"):
+        _query_terms("日本語の検索", tokenizer="janome_content")
+
+
+def test_query_terms_rejects_unknown_tokenizer():
+    with pytest.raises(ValueError, match="Unsupported query tokenizer"):
+        _query_terms("query", tokenizer="unknown")
+
+
 def test_score_nodes_chinese_substring_match():
     """Searching for '路由' should match a node with label containing '路由'."""
     G = nx.Graph()