diff --git a/README.md b/README.md index 0c14d207c..7c6f824db 100644 --- a/README.md +++ b/README.md @@ -262,7 +262,6 @@ Codex users also need `multi_agent = true` under `[features]` in `~/.codex/confi | `anthropic` | Anthropic Claude API (`--backend claude`, uses `ANTHROPIC_API_KEY`) | `uv tool install "graphifyy[anthropic]"` | | `bedrock` | AWS Bedrock (uses IAM, no API key) | `uv tool install "graphifyy[bedrock]"` | | `azure` | Azure OpenAI Service (`--backend azure`, uses `AZURE_OPENAI_API_KEY` + `AZURE_OPENAI_ENDPOINT`) | `uv tool install "graphifyy[openai]"` | -| `sql` | SQL schema extraction | `uv tool install "graphifyy[sql]"` | | `postgres` | Live PostgreSQL introspection (`--postgres DSN`) | `uv tool install "graphifyy[postgres]"` | | `dm` | BYOND DreamMaker `.dm`/`.dme` AST extraction (may need a C compiler + `python3-dev` if no wheel matches your platform) | `uv tool install "graphifyy[dm]"` | | `terraform` | Terraform / HCL `.tf`/`.tfvars`/`.hcl` AST extraction | `uv tool install "graphifyy[terraform]"` | @@ -272,6 +271,8 @@ Codex users also need `multi_agent = true` under `[features]` in `~/.codex/confi | `chinese` | Chinese query segmentation (jieba) | `uv tool install "graphifyy[chinese]"` | | `all` | Everything above | `uv tool install "graphifyy[all]"` | +> **SQL is no longer an extra.** `.sql` schema extraction is included in every default install (`tree-sitter-sql` is a core dependency). If you previously installed the `sql` extra or ran `pipx inject graphifyy tree-sitter-sql`, the normal upgrade (`uv tool upgrade graphifyy` or `pipx upgrade graphifyy`) is all you need — the `sql` extra remains as a harmless alias for older install scripts. + --- diff --git a/graphify/extract.py b/graphify/extract.py index ffc6153f8..a89647656 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -5073,7 +5073,9 @@ def add_existing_edge(edge: dict) -> None: # rather than falling back like Pascal does. Used by the #1745 warning in # extract() to tell the user which extra restores the language. _EXTRA_FOR_EXTENSION = { - ".sql": "sql", + # .sql is deliberately absent: tree-sitter-sql is a core dependency, so a + # missing grammar means a broken install, not a missing extra + # — the extractor's own error names the direct pip install that repairs it. ".tf": "terraform", ".tfvars": "terraform", ".hcl": "terraform", diff --git a/graphify/extractors/sql.py b/graphify/extractors/sql.py index a5dc18c36..3c0a94704 100644 --- a/graphify/extractors/sql.py +++ b/graphify/extractors/sql.py @@ -41,7 +41,9 @@ def extract_sql(path: Path, content: str | bytes | None = None) -> dict: # and surface the real exception in the latter case. if importlib.util.find_spec("tree_sitter_sql") is None: return {"nodes": [], "edges": [], - "error": "tree_sitter_sql not installed. Run: pip install tree-sitter-sql"} + "error": ("tree_sitter_sql not installed. It is a core dependency, " + "so this install is incomplete — reinstall " + "graphifyy or run: pip install tree-sitter-sql")} return {"nodes": [], "edges": [], "error": f"tree_sitter_sql is installed but failed to load: {e}"} diff --git a/pyproject.toml b/pyproject.toml index 15ea9dd57..29406fcfa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,13 @@ dependencies = [ "tree-sitter-fortran>=0.6,<0.8", "tree-sitter-bash>=0.23,<0.27", "tree-sitter-json>=0.23,<0.26", + # Core dependency (was the [sql] extra): .sql files are a mainstream + # corpus language and the extra-gated grammar made a default install + # silently skip them (#1745). Ships prebuilt abi3 wheels for every + # supported platform (win/macOS/Linux glibc+musl, x86_64+arm64) from + # 0.3.9, so no C toolchain is needed — the reason tree-sitter-dm stays + # optional does not apply here. + "tree-sitter-sql>=0.3.9,<0.4", ] [project.urls] @@ -74,7 +81,10 @@ anthropic = ["anthropic"] gemini = ["openai", "tiktoken"] openai = ["openai", "tiktoken"] chinese = ["jieba"] -sql = ["tree-sitter-sql"] +# tree-sitter-sql is now a core dependency. The extra is kept as +# an alias so existing `graphifyy[sql]` install commands and scripts keep +# resolving; it adds nothing beyond the core install. +sql = ["tree-sitter-sql>=0.3.9,<0.4"] # extract_pascal() uses tree-sitter-pascal for AST-quality extraction (more # accurate calls/inherits edges) and falls back to a regex extractor when it is # absent (#781), so this stays optional. Unlike tree-sitter-dm below, it ships diff --git a/tests/test_extract.py b/tests/test_extract.py index c9790e4ab..b8adbdb6b 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -3610,12 +3610,21 @@ def test_extract_no_warning_when_all_code_has_extractors(tmp_path, capsys): assert "no AST extractor" not in err -def test_extract_warns_when_sql_extra_missing(tmp_path, capsys, monkeypatch): +def test_extract_warns_when_sql_grammar_missing(tmp_path, capsys, monkeypatch): # #1745: .sql HAS a dispatch entry, so the #1689 warning can't fire, and # extract_sql returns an "error" result when tree-sitter-sql is absent, so - # the #1666 warning skips it too. The files must not vanish silently: - # extract() surfaces them with the [sql] extra named. + # the #1666 warning skips it too. The files must not vanish silently. + # The grammar is a core dependency, so absence means a broken + # install: the warning must name the direct repair, not a [sql] extra. + # The dev environment now ships the grammar, so simulate genuine absence + # by failing the import AND blanking find_spec for this one module. + import importlib.util as _ilu monkeypatch.setitem(sys.modules, "tree_sitter_sql", None) # import -> ImportError + _real_find_spec = _ilu.find_spec + monkeypatch.setattr( + _ilu, "find_spec", + lambda name, *a, **k: None if name == "tree_sitter_sql" else _real_find_spec(name, *a, **k), + ) s1 = tmp_path / "schema.sql"; s1.write_text("CREATE TABLE users (id INT);\n") s2 = tmp_path / "views.sql"; s2.write_text("CREATE VIEW v AS SELECT * FROM users;\n") py = tmp_path / "main.py"; py.write_text("def main():\n return 1\n") @@ -3625,7 +3634,8 @@ def test_extract_warns_when_sql_extra_missing(tmp_path, capsys, monkeypatch): assert "2 .sql file(s)" in err assert "tree_sitter_sql not installed" in err - assert 'graphifyy[sql]' in err + assert "core dependency" in err, "message must say the install is broken, not point at an extra" + assert "graphifyy[sql]" not in err, ".sql must not be hinted as an optional extra any more" assert "#1745" in err # the Python file still extracts normally labels = [n.get("label") for n in result["nodes"]] @@ -3639,7 +3649,7 @@ def test_extract_warns_when_sql_extra_missing(tmp_path, capsys, monkeypatch): def test_extract_failed_sources_empty_when_sql_installed(tmp_path): """#2543: successful extracts do not appear in failed_sources.""" - pytest.importorskip("tree_sitter_sql") + import tree_sitter_sql # noqa: F401 — core dependency; absence must FAIL, not skip s = tmp_path / "schema.sql"; s.write_text("CREATE TABLE users (id INT);\n") py = tmp_path / "main.py"; py.write_text("def main():\n return 1\n") result = extract([s, py], cache_root=tmp_path) @@ -3647,7 +3657,7 @@ def test_extract_failed_sources_empty_when_sql_installed(tmp_path): def test_extract_no_missing_dep_warning_when_sql_installed(tmp_path, capsys): - pytest.importorskip("tree_sitter_sql") + import tree_sitter_sql # noqa: F401 — core dependency; absence must FAIL, not skip s = tmp_path / "schema.sql"; s.write_text("CREATE TABLE users (id INT);\n") extract([s], cache_root=tmp_path) err = capsys.readouterr().err @@ -3661,7 +3671,7 @@ def test_extract_sql_reports_load_failure_not_missing(tmp_path, monkeypatch): # `pip install` — but surface the real load exception instead. import builtins from graphify.extractors.sql import extract_sql - pytest.importorskip("tree_sitter_sql") # find_spec must see it as installed + import tree_sitter_sql # noqa: F401 — core dependency; absence must FAIL, not skip # find_spec must see it as installed _orig_import = builtins.__import__ @@ -3682,7 +3692,7 @@ def test_extract_warns_sql_grammar_failed_to_load(tmp_path, capsys, monkeypatch) # grammar with the real cause and WITHOUT the misleading "install the extra" # hint, so the files are neither silently dropped nor sent to a no-op fix. import builtins - pytest.importorskip("tree_sitter_sql") + import tree_sitter_sql # noqa: F401 — core dependency; absence must FAIL, not skip _orig_import = builtins.__import__ diff --git a/tests/test_multilang.py b/tests/test_multilang.py index cb390eebc..f51adec36 100644 --- a/tests/test_multilang.py +++ b/tests/test_multilang.py @@ -450,39 +450,39 @@ def test_cache_miss_after_file_change(tmp_path): # ── SQL ─────────────────────────────────────────────────────────────────────── -def _extract_sql_or_skip(fixture: str = "sample.sql"): - pytest.importorskip("tree_sitter_sql") +def _extract_sql_required(fixture: str = "sample.sql"): + import tree_sitter_sql # noqa: F401 — core dependency; absence must FAIL, not skip return extract_sql(FIXTURES / fixture) def test_sql_finds_tables(): - r = _extract_sql_or_skip() + r = _extract_sql_required() labels = [n["label"] for n in r["nodes"]] assert any("users" in l for l in labels) assert any("organizations" in l for l in labels) def test_sql_finds_view(): - r = _extract_sql_or_skip() + r = _extract_sql_required() labels = [n["label"] for n in r["nodes"]] assert any("active_users" in l for l in labels) def test_sql_finds_function(): - r = _extract_sql_or_skip() + r = _extract_sql_required() labels = [n["label"] for n in r["nodes"]] assert any("get_user" in l for l in labels) def test_sql_emits_foreign_key_edge(): - r = _extract_sql_or_skip() + r = _extract_sql_required() relations = {e["relation"] for e in r["edges"]} assert "references" in relations def test_sql_emits_reads_from_edge(): - r = _extract_sql_or_skip() + r = _extract_sql_required() relations = {e["relation"] for e in r["edges"]} assert "reads_from" in relations def test_sql_no_dangling_edges(): - r = _extract_sql_or_skip() + r = _extract_sql_required() node_ids = {n["id"] for n in r["nodes"]} for e in r["edges"]: assert e["source"] in node_ids, f"dangling source: {e['source']}" @@ -495,7 +495,7 @@ def test_sql_cte_is_not_read_as_a_table(): with a same-named node from another language. The real table in the same FROM/JOIN must still resolve. """ - r = _extract_sql_or_skip("sample_cte.sql") + r = _extract_sql_required("sample_cte.sql") labels = [n["label"] for n in r["nodes"]] assert "levels" not in labels, "CTE name leaked into the graph as a table node" @@ -509,7 +509,7 @@ def test_sql_cte_is_not_read_as_a_table(): def test_sql_column_list_cte_is_not_read_as_a_table(tmp_path): """#2577: `WITH levels(a, b) AS (...)` — the name precedes a column list.""" - pytest.importorskip("tree_sitter_sql") + import tree_sitter_sql # noqa: F401 — core dependency; absence must FAIL, not skip p = tmp_path / "schema.sql" p.write_text( "CREATE TABLE users (id INT, role TEXT);\n" @@ -527,7 +527,7 @@ def test_sql_cte_shadows_same_named_table_within_its_statement(tmp_path): """#2577: inside the declaring statement the CTE shadows a real same-named table (SQL scoping), so v1's FROM binds to the CTE and emits nothing; v2 has no CTE in scope and reads the real table. Exactly one deterministic edge.""" - pytest.importorskip("tree_sitter_sql") + import tree_sitter_sql # noqa: F401 — core dependency; absence must FAIL, not skip p = tmp_path / "schema.sql" p.write_text( "CREATE TABLE levels (role TEXT);\n" @@ -546,7 +546,7 @@ def test_sql_subquery_cte_does_not_suppress_outer_real_table(tmp_path): """#2577 refinement: a WITH inside a subquery is scoped to that subquery only. A statement-wide pre-collect would also swallow the OUTER reference to the real `t2`, dropping a true edge — per-subtree scoping keeps it.""" - pytest.importorskip("tree_sitter_sql") + import tree_sitter_sql # noqa: F401 — core dependency; absence must FAIL, not skip p = tmp_path / "schema.sql" p.write_text( "CREATE TABLE t2 (id INT);\n" @@ -567,7 +567,7 @@ def test_sql_cte_never_binds_to_cross_language_symbol(tmp_path): so _rewire_unique_stub_nodes bound it to a same-named symbol from ANOTHER language (schema_v_roles -> ui_levels). With the CTE excluded, no reads_from edge may target a TypeScript node.""" - pytest.importorskip("tree_sitter_sql") + import tree_sitter_sql # noqa: F401 — core dependency; absence must FAIL, not skip sql = tmp_path / "schema.sql" sql.write_text( "CREATE TABLE users (id INT, role TEXT);\n" @@ -594,7 +594,7 @@ def test_sql_cross_file_fk_resolves_and_never_leaks_scan_path(tmp_path): minted a node-less id under the referencing file's own stem, which with absolute inputs leaked the machine path AND could never match the m1 definition, so prisma-style cross-migration FKs dangled.""" - pytest.importorskip("tree_sitter_sql") + import tree_sitter_sql # noqa: F401 — core dependency; absence must FAIL, not skip from graphify.ids import make_id m1 = tmp_path / "prisma" / "migrations" / "m1" @@ -647,7 +647,7 @@ def test_sql_cross_file_fk_resolves_and_never_leaks_scan_path(tmp_path): def test_sql_alter_table_fk_edge(): """ALTER TABLE ... FOREIGN KEY ... REFERENCES produces a references edge.""" - r = _extract_sql_or_skip("sample_alter_fk.sql") + r = _extract_sql_required("sample_alter_fk.sql") fk_edges = [e for e in r["edges"] if e["relation"] == "references"] assert len(fk_edges) >= 1 node_ids = {n["id"] for n in r["nodes"]} @@ -657,14 +657,14 @@ def test_sql_alter_table_fk_edge(): def test_sql_schema_qualified_names(): """Schema-qualified table names (Schema.Table) are preserved.""" - r = _extract_sql_or_skip("sample_schema_qualified.sql") + r = _extract_sql_required("sample_schema_qualified.sql") labels = [n["label"] for n in r["nodes"]] assert any("Sales.Customer" in l for l in labels) assert any("Sales.SalesOrder" in l for l in labels) def test_sql_schema_qualified_alter_fk(): """ALTER TABLE with schema-qualified names produces correct edges.""" - r = _extract_sql_or_skip("sample_schema_qualified.sql") + r = _extract_sql_required("sample_schema_qualified.sql") fk_edges = [e for e in r["edges"] if e["relation"] == "references"] assert len(fk_edges) >= 1 node_ids = {n["id"] for n in r["nodes"]} @@ -675,7 +675,7 @@ def test_sql_schema_qualified_alter_fk(): def test_sql_plpgsql_functions_survive_parse_errors(): """PL/pgSQL bodies make tree-sitter-sql emit ERROR nodes; the functions must still be extracted (#1910), without cascading into later statements.""" - r = _extract_sql_or_skip("sample_plpgsql.sql") + r = _extract_sql_required("sample_plpgsql.sql") labels = [n["label"] for n in r["nodes"]] # Both PL/pgSQL functions extracted, schema-qualified name kept whole assert "exposed.important_function()" in labels @@ -694,7 +694,7 @@ def test_sql_plpgsql_functions_survive_parse_errors(): def test_sql_plpgsql_clean_function_not_double_emitted(): """A cleanly-parsed LANGUAGE sql function in the same file is emitted once.""" - r = _extract_sql_or_skip("sample_plpgsql.sql") + r = _extract_sql_required("sample_plpgsql.sql") labels = [n["label"] for n in r["nodes"]] assert labels.count("plain_sql_fn()") == 1 # And nothing else is duplicated either @@ -712,7 +712,7 @@ def test_sql_quoted_plpgsql_routines_are_recovered(): with an *unquoted* name recovered fine, which is why the drop looked like it depended only on the body statement. """ - r = _extract_sql_or_skip("sample_plpgsql_quoted.sql") + r = _extract_sql_required("sample_plpgsql_quoted.sql") labels = [n["label"] for n in r["nodes"]] for name in ( "raise_exception_fn", @@ -727,7 +727,7 @@ def test_sql_quoted_plpgsql_routines_are_recovered(): def test_sql_quoted_plpgsql_file_stays_clean(): """The #2180 recovery must not add junk, duplicates, or drop the tables.""" - r = _extract_sql_or_skip("sample_plpgsql_quoted.sql") + r = _extract_sql_required("sample_plpgsql_quoted.sql") labels = [n["label"] for n in r["nodes"]] # Tables before and after the unparseable routines still extract. assert any("accounts" in l for l in labels) diff --git a/tests/test_sql_core_dependency.py b/tests/test_sql_core_dependency.py new file mode 100644 index 000000000..03dff2b38 --- /dev/null +++ b/tests/test_sql_core_dependency.py @@ -0,0 +1,100 @@ +"""SQL parsing is a core capability, not an optional extra. + +tree-sitter-sql used to live behind the [sql] extra, so a default +`uv tool install graphifyy` / `pipx install graphifyy` silently skipped every +.sql file in the corpus: the extractor bailed with an error, the #1745 warning +was the only signal, and until #2543 the failure was even stamped into the +incremental manifest. These tests pin the fix at the packaging layer so a +regression (the dependency sliding back into an extra) fails the suite rather +than resurfacing as a field report. +""" +from __future__ import annotations + +import sys +from pathlib import Path + +if sys.version_info >= (3, 11): + import tomllib +else: + import tomli as tomllib + +from graphify import extract as extractmod +from graphify.extract import extract + +REPO_ROOT = Path(__file__).resolve().parent.parent + + +def _pyproject() -> dict: + with open(REPO_ROOT / "pyproject.toml", "rb") as fh: + return tomllib.load(fh) + + +def test_tree_sitter_sql_is_a_core_dependency(): + """The grammar must be in [project.dependencies], not only in an extra.""" + deps = _pyproject()["project"]["dependencies"] + sql_deps = [d for d in deps if d.replace("_", "-").startswith("tree-sitter-sql")] + assert sql_deps, ( + "tree-sitter-sql is missing from [project.dependencies]; a default " + "install would silently skip every .sql file again (#1745)" + ) + + +def test_tree_sitter_sql_core_pin_stays_inside_supported_tree_sitter_range(): + """The core pin must carry an upper bound like every other grammar pin.""" + deps = _pyproject()["project"]["dependencies"] + (pin,) = [d for d in deps if d.replace("_", "-").startswith("tree-sitter-sql")] + assert "<" in pin, f"tree-sitter-sql core pin has no upper bound: {pin!r}" + + +def test_sql_extension_is_not_mapped_to_an_optional_extra(): + """#1745's hint map must not send users to a now-redundant [sql] extra.""" + assert ".sql" not in extractmod._EXTRA_FOR_EXTENSION, ( + "_EXTRA_FOR_EXTENSION still maps .sql to an extra; the 'install " + "graphifyy[sql]' hint is wrong now that the grammar is core" + ) + + +def test_tree_sitter_sql_imports_in_this_environment(): + """The dev environment itself must satisfy the core dependency.""" + import tree_sitter_sql # noqa: F401 + + +def test_sql_corpus_produces_structural_nodes_and_edges(tmp_path): + """Regression: a table/view/procedure corpus yields real structure. + + Guards the end-to-end path (dispatch -> tree-sitter parse -> node/edge + emission), not just the packaging declaration: representative DDL must + produce object nodes, `contains` edges from the file, a foreign-key + `references` edge, and `reads_from` edges from the view and procedure. + """ + schema = tmp_path / "schema.sql" + schema.write_text( + "CREATE TABLE organizations (\n" + " id INT PRIMARY KEY,\n" + " name TEXT NOT NULL\n" + ");\n" + "CREATE TABLE users (\n" + " id INT PRIMARY KEY,\n" + " org_id INT REFERENCES organizations(id)\n" + ");\n" + "CREATE VIEW active_users AS\n" + " SELECT * FROM users WHERE active = 1;\n" + "CREATE PROCEDURE prune_users()\n" + "BEGIN\n" + " DELETE FROM users WHERE id IN (SELECT id FROM active_users);\n" + "END;\n" + ) + + r = extract([schema]) + labels = {n["label"] for n in r["nodes"]} + assert {"organizations", "users", "active_users"} <= labels, labels + assert any(l.startswith("prune_users") for l in labels), labels + + relations = {e["relation"] for e in r["edges"]} + assert "contains" in relations, "file node must contain the SQL objects" + assert "references" in relations, "users.org_id FK must emit a references edge" + assert "reads_from" in relations, "view/procedure bodies must emit reads_from edges" + + node_ids = {n["id"] for n in r["nodes"]} + for e in r["edges"]: + assert e["source"] in node_ids, f"dangling edge source: {e['source']}" diff --git a/uv.lock b/uv.lock index 881314a5b..5275514ae 100644 --- a/uv.lock +++ b/uv.lock @@ -1120,6 +1120,7 @@ dependencies = [ { name = "tree-sitter-ruby" }, { name = "tree-sitter-rust" }, { name = "tree-sitter-scala" }, + { name = "tree-sitter-sql" }, { name = "tree-sitter-swift" }, { name = "tree-sitter-typescript" }, { name = "tree-sitter-verilog" }, @@ -1334,8 +1335,9 @@ requires-dist = [ { name = "tree-sitter-ruby", specifier = ">=0.23,<0.25" }, { name = "tree-sitter-rust", specifier = ">=0.23,<0.25" }, { name = "tree-sitter-scala", specifier = ">=0.23,<0.27" }, + { name = "tree-sitter-sql", specifier = ">=0.3.9,<0.4" }, { name = "tree-sitter-sql", marker = "extra == 'all'" }, - { name = "tree-sitter-sql", marker = "extra == 'sql'" }, + { name = "tree-sitter-sql", marker = "extra == 'sql'", specifier = ">=0.3.9,<0.4" }, { name = "tree-sitter-swift", specifier = ">=0.7,<0.9" }, { name = "tree-sitter-typescript", specifier = ">=0.23,<0.25" }, { name = "tree-sitter-verilog", specifier = ">=1.0,<2.0" },