Run SQL benchmarks with plan v2 (full stack vs develop)

.github/workflows/sql-bench-matrix.yml

@@ -103,6 +103,7 @@ jobs:
     env:
       VORTEX_EXPERIMENTAL_PATCHED_ARRAY: "1"
       FLAT_LAYOUT_INLINE_ARRAY_NODE: "1"
+      VORTEX_USE_PLAN_V2: "1"
       # Makes python output nicer
       COLUMNS: 120
     strategy:

Cargo.toml

@@ -23,6 +23,7 @@ members = [
     "vortex-btrblocks",
     "vortex-layout",
     "vortex-scan",
+    "vortex-scan-v2",
     "vortex-file",
     "vortex-ipc",
     "vortex",
@@ -323,6 +324,7 @@ vortex-proto = { version = "0.1.0", path = "./vortex-proto", default-features =
 vortex-row = { version = "0.1.0", path = "./vortex-row", default-features = false }
 vortex-runend = { version = "0.1.0", path = "./encodings/runend", default-features = false }
 vortex-scan = { version = "0.1.0", path = "./vortex-scan", default-features = false }
+vortex-scan-v2 = { version = "0.1.0", path = "./vortex-scan-v2", default-features = false }
 vortex-sequence = { version = "0.1.0", path = "encodings/sequence", default-features = false }
 vortex-session = { version = "0.1.0", path = "./vortex-session", default-features = false }
 vortex-sparse = { version = "0.1.0", path = "./encodings/sparse", default-features = false }

docs/developer-guide/index.md

@@ -23,6 +23,7 @@ internals/session
 internals/async-runtime
 internals/vtables
 internals/execution
+internals/scan-planning
 internals/stats-pruning
 internals/io
 internals/serialization

docs/developer-guide/internals/scan-planning.md

@@ -0,0 +1,68 @@
+# Scan Plans
+
+A scan plan is the physical plan for satisfying one scan query. It is a tree of physical operators
+over a row domain, describing the reads and derived work needed to produce that query's result.
+
+## Operators, not layout mirrors
+
+Plan operators describe *what work happens*, not *which layout produced it*. Their identity and
+operator-specific state are independent of the source layout kind. The complete plan node is not:
+its common lazy-child container can own hidden source state used to materialize individual children
+on demand.
+
+| Operator | Work |
+| --- | --- |
+| `SegmentScan` | read one segment and decode it to an array |
+| `Concat` | concatenate its children row-wise |
+| `Pack` | assemble a struct from one child per field, plus optional validity |
+| `Take` | index `values` by `codes` |
+| `ListPack` | assemble a list from elements and offsets, plus optional validity |
+| `Eval` | apply an expression to its child |
+| `RowIdx` | offset row numbers into the file's row domain |
+
+Naming operators for what they compute is what lets one rule cover every case. `Concat` of
+`Concat` flattens on shape alone, and `Take` over `SegmentScan` is the dictionary pushdown,
+regardless of the source layout.
+
+The stored layout tree describes all physical data in a file. A plan is query-specific: it is built
+from that tree for one projection, filter, and row domain. Different queries over the same file can
+therefore produce different plans.
+
+## Optimization
+
+Child replacement is implemented by the common plan container rather than by every operator. It
+replaces the external child container, clones `PlanData`, then invokes the operator's
+`PlanVTable::with_children` callback to validate the new children and refresh derived caches such
+as `Concat` row offsets. Rules therefore rewrite the generic tree without reconstructing common
+plan fields inside each operator.
+
+Optimization rewrites the initial tree so that each expression is evaluated as close as possible to
+the physical data that can satisfy it. Every rewrite must preserve the query result, including its
+dtype, row domain, row order, row identity, null behavior, and observable errors.
+
+Planning does not read segment data. It constructs and optimizes a description of the work that a
+later execution stage will perform.
+
+## Vtables
+
+Each operator is a small vtable type implementing `PlanVTable`, paired with a `Plan<V>` container
+over a shared `PlanRef`. `PlanRef` points to one allocation whose ordinary fields hold the operator
+ID, dtype, row count, and lazy children. Only the unsized tail containing the vtable and
+`V::PlanData` is erased behind `dyn DynPlan`, so common-field reads do not use dynamic dispatch.
+`Plan<V>` provides typed access to that operator data through `Deref`.
+
+`PlanVTable` also carries `id` and a `Metadata` codec. Operators with no unrecoverable state
+already serialize their metadata; the ones holding a read context or a bound expression return
+`None` until those codecs exist.
+
+## Execution
+
+Each operator executes over a row range and selection mask. `SegmentScan` reads its segment,
+structural operators combine their children, and `Eval` applies the remaining derived work.
+`vortex-scan-v2` copies the existing scan orchestration around this API, so the original
+`LayoutReader` scanner is untouched while the plan-native path is developed.
+
+## Future work
+
+Still to come: a plan registry and foreign operator placeholder so third-party operators survive
+a round trip, and a serialization envelope.

vortex-array/src/expr/bound_expression.rs

@@ -226,6 +226,22 @@ impl BoundExpression {
         matches!(self.kind, BoundKind::Root)
     }
 
+    /// Return whether every scope root in this expression has `dtype`.
+    ///
+    /// Expressions without a scope root, such as literals, match every dtype.
+    pub fn is_root_bound_to(&self, dtype: &DType) -> bool {
+        let mut is_bound_to = true;
+        pre_order_visit_down(self, |node| {
+            if node.is_root() && node.dtype() != dtype {
+                is_bound_to = false;
+                return Ok(TraversalOrder::Stop);
+            }
+            Ok(TraversalOrder::Continue)
+        })
+        .vortex_expect("bound expression traversal cannot not fail");
+        is_bound_to
+    }
+
     /// Return an expression that proves this predicate is definitely false from statistics.
     pub fn falsify(&self, session: &VortexSession) -> VortexResult<Option<BoundExpression>> {
         StatsRewriteCtx::new(session).falsify(self)
@@ -363,6 +379,20 @@ mod tests {
         Ok(())
     }
 
+    #[test]
+    fn bound_to_checks_every_root() -> VortexResult<()> {
+        let dtype = struct_dtype();
+        let bound = eq(col("a"), col("a")).bind(&dtype)?;
+        assert!(bound.is_root_bound_to(&dtype));
+        assert!(!bound.is_root_bound_to(&DType::Bool(Nullability::NonNullable)));
+        assert!(
+            lit(true)
+                .bind(&dtype)?
+                .is_root_bound_to(&DType::Bool(Nullability::NonNullable))
+        );
+        Ok(())
+    }
+
     #[test]
     fn bound_display_matches_unbound() -> VortexResult<()> {
         for expr in [root(), col("a"), eq(col("a"), lit(1_i32)), lit(true)] {

vortex-datafusion/Cargo.toml

@@ -38,6 +38,7 @@ tokio-stream = { workspace = true }
 tracing = { workspace = true, features = ["std", "attributes"] }
 vortex = { workspace = true, features = ["object_store", "tokio", "files"] }
 vortex-arrow = { workspace = true }
+vortex-scan-v2 = { workspace = true }
 vortex-utils = { workspace = true, features = ["dashmap"] }
 
 [dev-dependencies]