From 0390de82515d3ec06298dda0e31851267b673892 Mon Sep 17 00:00:00 2001 From: prrao87 Date: Fri, 30 Oct 2026 10:47:23 -0400 Subject: [PATCH 1/3] docs: add Rust tabs to every getting started example Each Python snippet on the getting started page now has a Rust tab with the same data and query. The Rust code lives in a getting_started function of the embedded_query example, so CI compiles and runs it. The CSR tab now uses the same in-memory graph instead of the on-disk one, so both tabs print the same answer. --- crates/lance-csr/examples/embedded_query.rs | 133 +++++++++++++++- docs/getting-started.md | 167 ++++++++++++-------- 2 files changed, 230 insertions(+), 70 deletions(-) diff --git a/crates/lance-csr/examples/embedded_query.rs b/crates/lance-csr/examples/embedded_query.rs index 6cb6493..71cf107 100644 --- a/crates/lance-csr/examples/embedded_query.rs +++ b/crates/lance-csr/examples/embedded_query.rs @@ -2,9 +2,9 @@ //! //! Writes a small social graph as two Lance datasets in a temporary directory, opens //! them in a CSR-backed engine, runs a two-hop query, appends an edge and picks it up -//! with `refresh()`, runs a query under a deadline, and repeats the home page's in-memory -//! Arrow example. The documentation includes this -//! file section by section (the `--8<--` markers), so keep it compiling and its printed +//! with `refresh()`, runs a query under a deadline, and repeats the in-memory Arrow +//! examples of the home page and the getting started guide. The documentation includes +//! this file section by section (the `--8<--` markers), so keep it compiling and its printed //! output in step. //! //! ```bash @@ -20,7 +20,7 @@ use arrow_array::{ArrayRef, Int64Array, RecordBatch, RecordBatchIterator, String use arrow_schema::{DataType, Field, Schema}; use lance::dataset::{Dataset, WriteMode, WriteParams}; use lance_csr::query::with_execution_deadline; -use lance_csr::{CypherEngine, EngineSource, GraphConfig}; +use lance_csr::{CypherEngine, CypherQuery, EngineSource, GraphConfig}; use tokio::time::{timeout_at, Instant}; fn knows(src: Vec, dst: Vec) -> lance_csr::Result { @@ -178,5 +178,130 @@ async fn main() -> lance_csr::Result<()> { // | Carol | // +--------+ // --8<-- [end:quickstart] + + getting_started().await +} + +/// The Rust tabs of the getting started guide, one section per Python snippet. +async fn getting_started() -> lance_csr::Result<()> { + // --8<-- [start:gs_datasets] + let int = |values: Vec| Arc::new(Int64Array::from(values)) as ArrayRef; + let text = |values: Vec<&str>| Arc::new(StringArray::from(values)) as ArrayRef; + + let datasets = HashMap::from([ + ( + "Person".to_string(), + RecordBatch::try_from_iter([ + ("id", int(vec![1, 2, 3, 4])), + ("name", text(vec!["Alice", "Bob", "Carol", "David"])), + ("age", int(vec![28, 34, 29, 42])), + ])?, + ), + ( + "City".to_string(), + RecordBatch::try_from_iter([ + ("id", int(vec![10, 20])), + ("name", text(vec!["London", "Sydney"])), + ])?, + ), + ( + "KNOWS".to_string(), + RecordBatch::try_from_iter([ + ("src", int(vec![1, 1, 2, 3])), + ("dst", int(vec![2, 3, 4, 4])), + ])?, + ), + ( + "LIVES_IN".to_string(), + RecordBatch::try_from_iter([ + ("src", int(vec![1, 2, 3, 4])), + ("dst", int(vec![10, 10, 20, 20])), + ])?, + ), + ]); + // --8<-- [end:gs_datasets] + + // --8<-- [start:gs_config] + let config = GraphConfig::builder() + .node_table("Person", "id") + .node_table("City", "id") + .rel_table("KNOWS", "Person", "Person", "src", "dst") + .rel_table("LIVES_IN", "Person", "City", "src", "dst") + .build()?; + // --8<-- [end:gs_config] + + // --8<-- [start:gs_query] + // The engine reads each dataset as an `EngineSource`; `false` leaves the CSR off. + let sources: HashMap<_, _> = datasets + .iter() + .map(|(name, batch)| (name.clone(), EngineSource::Arrow(batch.clone()))) + .collect(); + let engine = CypherEngine::new(config.clone(), sources.clone(), false).await?; + + let result = engine + .execute( + "MATCH (p:Person)-[:LIVES_IN]->(c:City {name: 'London'}) + RETURN p.name AS name, p.age AS age + ORDER BY age", + HashMap::new(), + ) + .await?; + print_batches(&[result])?; + // +-------+-----+ + // | name | age | + // +-------+-----+ + // | Alice | 28 | + // | Bob | 34 | + // +-------+-----+ + // --8<-- [end:gs_query] + + // --8<-- [start:gs_params] + let older = engine + .execute( + "MATCH (p:Person) WHERE p.age >= $min_age RETURN p.name AS name ORDER BY name", + HashMap::from([("min_age".to_string(), serde_json::json!(30))]), + ) + .await?; + print_batches(&[older])?; + // +-------+ + // | name | + // +-------+ + // | Bob | + // | David | + // +-------+ + // --8<-- [end:gs_params] + + // --8<-- [start:gs_csr] + let engine = CypherEngine::new(config.clone(), sources, true).await?; + + let friends_of_friends = engine + .execute( + "MATCH (a:Person {name: 'Alice'})-[:KNOWS]->()-[:KNOWS]->(c:Person) + RETURN DISTINCT c.name AS friend_of_friend", + HashMap::new(), + ) + .await?; + print_batches(&[friends_of_friends])?; + // +------------------+ + // | friend_of_friend | + // +------------------+ + // | David | + // +------------------+ + // --8<-- [end:gs_csr] + + // --8<-- [start:gs_one_off] + let result = + CypherQuery::new("MATCH (p:Person) WHERE p.age > 30 RETURN p.name AS name ORDER BY name")? + .with_config(config) + .execute(datasets, None) + .await?; + print_batches(&[result])?; + // +-------+ + // | name | + // +-------+ + // | Bob | + // | David | + // +-------+ + // --8<-- [end:gs_one_off] Ok(()) } diff --git a/docs/getting-started.md b/docs/getting-started.md index 5975084..d6028ba 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -55,70 +55,103 @@ A graph in lance-csr is a set of ordinary tables plus a `GraphConfig` that says which are relationships, and which columns link them. Here are people, the cities they live in, and who knows whom: -```python -import pyarrow as pa - -datasets = { - "Person": pa.table({ - "id": [1, 2, 3, 4], - "name": ["Alice", "Bob", "Carol", "David"], - "age": [28, 34, 29, 42], - }), - "City": pa.table({"id": [10, 20], "name": ["London", "Sydney"]}), - "KNOWS": pa.table({"src": [1, 1, 2, 3], "dst": [2, 3, 4, 4]}), - "LIVES_IN": pa.table({"src": [1, 2, 3, 4], "dst": [10, 10, 20, 20]}), -} -``` +=== "Python" + + ```python + import pyarrow as pa + + datasets = { + "Person": pa.table({ + "id": [1, 2, 3, 4], + "name": ["Alice", "Bob", "Carol", "David"], + "age": [28, 34, 29, 42], + }), + "City": pa.table({"id": [10, 20], "name": ["London", "Sydney"]}), + "KNOWS": pa.table({"src": [1, 1, 2, 3], "dst": [2, 3, 4, 4]}), + "LIVES_IN": pa.table({"src": [1, 2, 3, 4], "dst": [10, 10, 20, 20]}), + } + ``` + +=== "Rust" + + ```rust + --8<-- "crates/lance-csr/examples/embedded_query.rs:gs_datasets" + ``` Each key of `datasets` is a node label or relationship type. Now describe how they connect: -```python -from lance_csr import GraphConfig - -config = ( - GraphConfig.builder() - .node_table("Person", "id") # (1)! - .node_table("City", "id") - .rel_table("KNOWS", "Person", "Person", "src", "dst") # (2)! - .rel_table("LIVES_IN", "Person", "City", "src", "dst") - .build() -) -``` +=== "Python" -1. A node table: the label `Person`, read from the dataset named `Person`, identified by its `id` column. -2. A relationship table: `KNOWS` edges go from `Person` to `Person`; the `src` column holds the source - node's `id` and `dst` the target's. + ```python + from lance_csr import GraphConfig + + config = ( + GraphConfig.builder() + .node_table("Person", "id") # (1)! + .node_table("City", "id") + .rel_table("KNOWS", "Person", "Person", "src", "dst") # (2)! + .rel_table("LIVES_IN", "Person", "City", "src", "dst") + .build() + ) + ``` + + 1. A node table: the label `Person`, read from the dataset named `Person`, identified by its `id` column. + 2. A relationship table: `KNOWS` edges go from `Person` to `Person`; the `src` column holds the source + node's `id` and `dst` the target's. + +=== "Rust" + + ```rust + --8<-- "crates/lance-csr/examples/embedded_query.rs:gs_config" + ``` ## Run a query `CypherEngine` registers the datasets once and runs as many queries as you like against them: -```python -from lance_csr import CypherEngine +=== "Python" -engine = CypherEngine(config, datasets) + ```python + from lance_csr import CypherEngine -result = engine.execute(""" - MATCH (p:Person)-[:LIVES_IN]->(c:City {name: 'London'}) - RETURN p.name AS name, p.age AS age - ORDER BY age -""") -print(result.to_pylist()) -# [{'name': 'Alice', 'age': 28}, {'name': 'Bob', 'age': 34}] -``` + engine = CypherEngine(config, datasets) + + result = engine.execute(""" + MATCH (p:Person)-[:LIVES_IN]->(c:City {name: 'London'}) + RETURN p.name AS name, p.age AS age + ORDER BY age + """) + print(result.to_pylist()) + # [{'name': 'Alice', 'age': 28}, {'name': 'Bob', 'age': 34}] + ``` + +=== "Rust" + + ```rust + --8<-- "crates/lance-csr/examples/embedded_query.rs:gs_query" + ``` Results come back as a [`pyarrow.Table`](https://arrow.apache.org/docs/python/generated/pyarrow.Table.html), -so `to_pylist()`, `to_pydict()`, `to_pandas()` and Polars' `pl.from_arrow()` all work. +so `to_pylist()`, `to_pydict()`, `to_pandas()` and Polars' `pl.from_arrow()` all work. In Rust, `execute` +returns an Arrow `RecordBatch`. Pass values with `$parameters` instead of formatting them into the query string: -```python -older = engine.execute( - "MATCH (p:Person) WHERE p.age >= $min_age RETURN p.name AS name ORDER BY name", - {"min_age": 30}, -) -print(older.to_pydict()) # {'name': ['Bob', 'David']} -``` +=== "Python" + + ```python + older = engine.execute( + "MATCH (p:Person) WHERE p.age >= $min_age RETURN p.name AS name ORDER BY name", + {"min_age": 30}, + ) + print(older.to_pydict()) # {'name': ['Bob', 'David']} + ``` + +=== "Rust" + + ```rust + --8<-- "crates/lance-csr/examples/embedded_query.rs:gs_params" + ``` ## Go multi-hop with the CSR @@ -139,18 +172,12 @@ patterns then become array lookups instead of joins: === "Rust" - The same engine is a Rust crate (`cargo add lance-csr`). This example reads `Person` and `KNOWS` from - Lance datasets on disk, where Alice knows only Bob, so the answer differs from the Python tab. The third - argument of `CypherEngine::new` is `use_csr`. + The third argument of `CypherEngine::new` is `use_csr`. ```rust - --8<-- "crates/lance-csr/examples/embedded_query.rs:engine" - - --8<-- "crates/lance-csr/examples/embedded_query.rs:query" + --8<-- "crates/lance-csr/examples/embedded_query.rs:gs_csr" ``` - [Using from Rust](rust.md) has the full, runnable example, along with refresh, errors and timeouts. - !!! tip "When should I turn the CSR on?" Turn it on when your queries traverse relationships, and especially for two or more hops. Node keys and @@ -161,16 +188,24 @@ patterns then become array lookups instead of joins: For a single query you don't plan to repeat, `CypherQuery` skips the engine: -```python -from lance_csr import CypherQuery - -result = ( - CypherQuery("MATCH (p:Person) WHERE p.age > 30 RETURN p.name AS name ORDER BY name") - .with_config(config) - .execute(datasets) -) -print(result.to_pydict()) # {'name': ['Bob', 'David']} -``` +=== "Python" + + ```python + from lance_csr import CypherQuery + + result = ( + CypherQuery("MATCH (p:Person) WHERE p.age > 30 RETURN p.name AS name ORDER BY name") + .with_config(config) + .execute(datasets) + ) + print(result.to_pydict()) # {'name': ['Bob', 'David']} + ``` + +=== "Rust" + + ```rust + --8<-- "crates/lance-csr/examples/embedded_query.rs:gs_one_off" + ``` `CypherQuery` rebuilds its context on every `execute()` call, so prefer `CypherEngine` whenever you run more than one query against the same data. From ec5854c2f488cb9fd872e42dad35c3aa5bac471e Mon Sep 17 00:00:00 2001 From: prrao87 Date: Fri, 30 Oct 2026 10:47:23 -0400 Subject: [PATCH 2/3] docs: explain how to keep the CSR fresh Add a how-to to the CSR page: refresh after writes, check csr_stats(), and what forces a full rebuild instead of a delta, with Python and Rust examples. --- docs/csr.md | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/docs/csr.md b/docs/csr.md index 4bcfb21..97fc195 100644 --- a/docs/csr.md +++ b/docs/csr.md @@ -264,6 +264,53 @@ flowchart TD The 25% threshold is fixed: lookups pay a little for every pending delta edge, and a rebuild clears them. `python/benchmarks/csr_delta_accumulated.py` measures that trade-off. +### How to keep the CSR fresh + +The engine doesn't watch your datasets for changes. Keeping the CSR in step with the data is up to you: + +1. **Write with any Lance writer**: `pylance`, LanceDB or another process. +2. **Call `refresh()` after the writes your queries should see.** When a writer updates several tables, call it + after all of them commit (see [snapshots are per dataset](lance.md#lazy-version-pinned-reads)). A refresh + with nothing new reuses the existing CSR, so calling it on a timer or before each batch of queries is fine. +3. **Check what it did with [`csr_stats()`](#csr_stats)**: `update` says whether the CSR was `reused`, updated + with a `delta` or fully rebuilt (`rebuild`). + +=== "Python" + + ```python + import lance + + lance.write_dataset(datasets["Person"], "person.lance") + lance.write_dataset(datasets["KNOWS"], "knows.lance") + engine = CypherEngine( + config, + {"Person": lance.dataset("person.lance"), "KNOWS": lance.dataset("knows.lance")}, + use_csr=True, + ) + + # David now knows Alice. The engine doesn't see the write until refresh(). + lance.write_dataset(pa.table({"src": [4], "dst": [1]}), "knows.lance", mode="append") + engine.refresh() + + stats = engine.csr_stats()["KNOWS"] + print(stats["edges"], stats["delta_edges"], stats["update"]) # 5 1 delta + ``` + +=== "Rust" + + This continues the on-disk example from [Using from Rust](rust.md), where Bob gains an edge to Alice. + + ```rust + --8<-- "crates/lance-csr/examples/embedded_query.rs:refresh" + ``` + +A refresh can only apply a delta when it can compare the version the engine is pinned to with the latest one. +If you overwrite a relationship dataset, or remove the pinned version's files (for example with +`cleanup_old_versions()`) before refreshing, the next refresh rebuilds that CSR in full. Refresh more often +than you clean up old versions to keep updates incremental. + +In-memory PyArrow tables never change. To query new in-memory data, build a new engine. + ## Should I turn it on? | Situation | Recommendation | From 4291b2db74f952a609d81be76ec3936f1c06a38a Mon Sep 17 00:00:00 2001 From: prrao87 Date: Fri, 30 Oct 2026 10:47:23 -0400 Subject: [PATCH 3/3] docs: group the API references and move benchmarks up the sidebar --- zensical.toml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/zensical.toml b/zensical.toml index 150582a..9d1d1ae 100644 --- a/zensical.toml +++ b/zensical.toml @@ -25,6 +25,7 @@ extra_css = ["stylesheets/extra.css"] nav = [ { "Home" = "index.md" }, { "Getting started" = "getting-started.md" }, + { "Benchmarks" = "benchmarks.md" }, { "Guide" = [ { "Graph schema" = "graph-schema.md" }, { "Querying with Cypher" = "cypher.md" }, @@ -35,9 +36,10 @@ nav = [ ] }, { "Using from Rust" = "rust.md" }, { "How it works" = "execution.md" }, - { "Python API reference" = "api-reference.md" }, - { "Rust API reference" = "https://docs.rs/lance-csr" }, - { "Benchmarks" = "benchmarks.md" }, + { "API reference" = [ + { "Python" = "api-reference.md" }, + { "Rust (docs.rs)" = "https://docs.rs/lance-csr" }, + ] }, { "Contributing" = [ { "Contributing" = "contributing.md" }, { "Project structure" = "project_structure.md" },