From 9b8886e3f6aadfdbd03c7951bb4eb2a0f8e1b328 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sat, 3 Oct 2026 21:00:36 +0000 Subject: [PATCH 01/17] feat(planner): #509 Stage 2 physical candidates and Stage 3 plan selection (MVP) Stage 2 implements one logical candidate physically: exact Aggregate{[TopK]} becomes Limit(Sort) per group, summaries stay build -> estimate, every node runs at query time, and the timed roots export as one PhysicalASAPDAG. Stage 3 is the only stage that computes cost. It rejects candidates whose summary misses its query's accuracy target (accuracy model, analytical guarantee) or uses Count-Min over weights not proven non-negative, prices every valid candidate node by node with analytical_cost (illustrative statistics), selects the cheapest and reports the rest as costlier. Co-Authored-By: Claude Opus 5.5 --- crates/asap-aware-mapping/src/lib.rs | 6 + .../src/physical_candidates.rs | 278 ++++++++ .../asap-aware-mapping/src/plan_selection.rs | 655 ++++++++++++++++++ 3 files changed, 939 insertions(+) create mode 100644 crates/asap-aware-mapping/src/physical_candidates.rs create mode 100644 crates/asap-aware-mapping/src/plan_selection.rs diff --git a/crates/asap-aware-mapping/src/lib.rs b/crates/asap-aware-mapping/src/lib.rs index 41b58551..62b1400f 100644 --- a/crates/asap-aware-mapping/src/lib.rs +++ b/crates/asap-aware-mapping/src/lib.rs @@ -215,3 +215,9 @@ pub mod maintained_population; /// Stage 1, Pass 1: per-computation candidate generation. No execution timing is /// assigned: that is a Stage 2 materialization decision. pub mod per_computation_candidates; + +/// #509 Stage 2 MVP: physical operator implementation, all at query time. +pub mod physical_candidates; + +/// #509 Stage 3 MVP: accuracy check, analytical pricing, cheapest valid plan. +pub mod plan_selection; diff --git a/crates/asap-aware-mapping/src/physical_candidates.rs b/crates/asap-aware-mapping/src/physical_candidates.rs new file mode 100644 index 00000000..a581e4e0 --- /dev/null +++ b/crates/asap-aware-mapping/src/physical_candidates.rs @@ -0,0 +1,278 @@ +//! #509 Stage 2 (MVP): physical operator implementation of one logical +//! candidate. No materialization choice is made: every node runs at query +//! time, so each logical candidate yields exactly one physical candidate. +//! +//! The runtime has one implementation per logical operator except exact +//! top-k, which it cannot run as `Aggregate{[TopK]}`. Stage 2 records that +//! choice in the DAG by rewriting it to a per-group sort followed by a +//! per-group limit, the shape the PromQL frontend uses for generic `topk`. +//! A summary needs no rewrite: `SummaryAgg` → `SummaryEstimate` already is +//! build → estimate. +use std::collections::HashMap; +use std::rc::Rc; + +use asap_types::ir::export::{compile_physical_asap_workload_with_node_ids, PhysicalASAPDAG}; +use asap_types::ir::{ + apply_materialization_timings, ASAPOp, MaterializationAssignment, NonASAPOp, Operator, + OperatorNode, ScalarExpr, SchemaDerivationError, SortKey, TimingMemo, +}; +use asap_types::post_asap::ExecutionDataStateError; +use asap_types::pre_asap::column_resolution::resolve_column_ref; +use asap_types::pre_asap::expr_ir::ColumnRef; +use asap_types::pre_asap::{AggIntent, Reduction}; +use thiserror::Error; + +/// One Stage 2 candidate, derived from exactly one Stage 1 candidate. +/// `roots` are the timed operator roots (one per query, in workload order) +/// that `dag` exports. +#[derive(Debug, Clone)] +pub struct PhysicalCandidate { + pub id: String, + pub from_logical: String, + pub label: String, + pub roots: Vec>, + pub dag: PhysicalASAPDAG, +} + +#[derive(Debug, Error)] +pub enum Stage2Error { + #[error(transparent)] + Structure(#[from] SchemaDerivationError), + #[error(transparent)] + Timing(#[from] ExecutionDataStateError), + #[error("exact top-k has no sortable value column: {0}")] + NoValueColumn(String), + #[error("maintained populations run at ingestion time, which Stage 2 does not plan yet")] + IngestionTimeOnly, +} + +/// Implement every operator of one logical candidate and time it at query +/// time. `id` and `label` are left empty for the caller to name. Sharing +/// between `roots` is preserved: one memo serves the whole workload. +pub fn stage2_physical( + from_logical: &str, + roots: &[Rc], +) -> Result { + let mut memo = HashMap::new(); + let implemented = roots + .iter() + .map(|root| implement(root, &mut memo)) + .collect::, _>>()?; + reject_maintained_populations(&implemented)?; + // The default assignment computes every summary state at query time. + let assignment = MaterializationAssignment::default(); + let mut timing = TimingMemo::new(); + let timed = implemented + .iter() + .map(|root| apply_materialization_timings(root, &assignment, &mut timing)) + .collect::, _>>()?; + let dag = compile_physical_asap_workload_with_node_ids(&timed)?.dag; + Ok(PhysicalCandidate { + id: String::new(), + from_logical: from_logical.to_string(), + label: String::new(), + roots: timed, + dag, + }) +} + +/// A maintained population always runs at ingestion time, which Stage 2 +/// does not plan yet. +fn reject_maintained_populations(roots: &[Rc]) -> Result<(), Stage2Error> { + let maintained = roots.iter().flat_map(OperatorNode::reachable).any(|node| { + matches!( + node.operator, + Operator::ASAP(ASAPOp::MaintainPopulation { .. }) + ) + }); + if maintained { + return Err(Stage2Error::IngestionTimeOnly); + } + Ok(()) +} + +type Memo = HashMap<*const OperatorNode, Rc>; + +fn implement(node: &Rc, memo: &mut Memo) -> Result, Stage2Error> { + if let Some(done) = memo.get(&Rc::as_ptr(node)) { + return Ok(done.clone()); + } + for child in node.children() { + implement(child, memo)?; + } + let implemented = match node.non_asap() { + Some(NonASAPOp::Aggregate { + reduction: Reduction::Reduce(keys), + measures, + filters, + having: None, + child, + .. + }) if filters.iter().all(Option::is_none) => match measures.as_slice() { + [AggIntent::TopK { k, .. }] => { + let child = memo[&Rc::as_ptr(child)].clone(); + let value = resolve_column_ref(&ColumnRef::SampleValue, &child.schema) + .map_err(|e| Stage2Error::NoValueColumn(e.to_string()))?; + let sorted = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Sort { + keys: vec![SortKey { + expr: ScalarExpr::Column(value), + ascending: false, + nulls_first: false, + }], + partition_by: keys.clone(), + child, + }))?; + OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Limit { + n: Some(*k), + offset: 0, + partition_by: keys.clone(), + child: sorted, + }))? + } + _ => rebuilt(node, memo)?, + }, + _ => rebuilt(node, memo)?, + }; + memo.insert(Rc::as_ptr(node), implemented.clone()); + Ok(implemented) +} + +/// `node` over its implemented children; the same `Rc` when none changed. +fn rebuilt(node: &Rc, memo: &Memo) -> Result, Stage2Error> { + let changed = node + .children() + .iter() + .any(|child| !Rc::ptr_eq(child, &memo[&Rc::as_ptr(child)])); + Ok(if changed { + Rc::new(node.map_children(|child| memo[&Rc::as_ptr(child)].clone())?) + } else { + node.clone() + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_support::lower_promql; + use asap_types::ir::export::PhysicalASAPOperatorPayload as Payload; + use asap_types::ir::export::{NonASAPOpKind, PhysicalASAPNodeId}; + use asap_types::ir::QueryRoot; + use asap_types::post_asap::ExecutionTiming; + use asap_types::types::AccuracyTarget; + + /// Exact `topk by (job)` becomes Limit(Sort) partitioned by `job`, and a + /// child shared by two roots stays one node (one `Rc`, one DAG node). + #[test] + fn exact_topk_becomes_sort_then_limit_and_keeps_shared_child() { + let exact = AccuracyTarget::Exact; + let topk = lower_promql("topk by (job) (10, sum_over_time(m[1m]))", exact.clone()); + let sum = lower_promql("sum by (job) (sum_over_time(m[1m]))", exact); + // Share Q2's per-series child with Q1, as Stage 1 sharing would. + let NonASAPOp::Aggregate { child: shared, .. } = topk.expect_non_asap() else { + panic!("topk aggregate") + }; + let sum = Rc::new(sum.map_children(|_| shared.clone()).unwrap()); + let candidate = stage2_physical("L1", &[topk, sum]).unwrap(); + + let NonASAPOp::Limit { + n: Some(10), + partition_by, + child: sort, + .. + } = candidate.roots[0].expect_non_asap() + else { + panic!("limit root") + }; + assert!(!partition_by.keys().is_empty()); + let NonASAPOp::Sort { + child: below_sort, + partition_by: sort_partition, + .. + } = sort.expect_non_asap() + else { + panic!("sort below limit") + }; + assert_eq!(sort_partition, partition_by); + let NonASAPOp::Aggregate { + child: below_sum, .. + } = candidate.roots[1].expect_non_asap() + else { + panic!("sum root") + }; + assert!(Rc::ptr_eq(below_sort, below_sum)); + + let dag = &candidate.dag; + let kind = |id: PhysicalASAPNodeId| match &dag + .nodes + .iter() + .find(|n| n.id == id) + .unwrap() + .payload + { + Payload::Relational { + operator: NonASAPOpKind::Sort { .. }, + } => "sort", + Payload::Relational { + operator: NonASAPOpKind::Limit { .. }, + } => "limit", + _ => "other", + }; + let sort_id = dag + .edges + .iter() + .find(|e| kind(e.producer) == "sort" && e.consumer == dag.roots[0]) + .expect("sort -> limit edge") + .producer; + let producer_of = |consumer| { + dag.edges + .iter() + .find(|e| e.consumer == consumer) + .unwrap() + .producer + }; + assert_eq!(producer_of(sort_id), producer_of(dag.roots[1])); + } + + /// With summaries chosen, every node, the summary build included, runs at + /// query time. + #[test] + fn everything_runs_at_query_time() { + let root = lower_promql( + "topk by (job) (10, sum_over_time(m[1m]))", + AccuracyTarget::EpsilonDelta { + epsilon: 0.01, + delta: 0.001, + }, + ); + let inventory = crate::per_computation_candidates::generate_per_computation_candidates( + vec![(0, QueryRoot::Operator(root))], + ) + .unwrap(); + // A summary (the last alternative) for every target. + let choice: Vec<_> = inventory + .targets + .iter() + .map(|t| t.candidates.len() - 1) + .collect(); + let roots: Vec<_> = + crate::per_computation_candidates::compose_logical_candidate(&inventory, &choice) + .unwrap() + .into_iter() + .map(|(_, root)| match root { + QueryRoot::Operator(node) => node, + QueryRoot::Scalar(_) => panic!("operator root"), + }) + .collect(); + let candidate = stage2_physical("L1", &roots).unwrap(); + assert!(candidate + .dag + .nodes + .iter() + .any(|n| matches!(n.payload, Payload::SummaryAgg { .. }))); + assert!(candidate + .dag + .nodes + .iter() + .all(|n| n.output_state.timing == ExecutionTiming::QueryTime)); + } +} diff --git a/crates/asap-aware-mapping/src/plan_selection.rs b/crates/asap-aware-mapping/src/plan_selection.rs new file mode 100644 index 00000000..9909a506 --- /dev/null +++ b/crates/asap-aware-mapping/src/plan_selection.rs @@ -0,0 +1,655 @@ +//! #509 Stage 3 (MVP): plan selection, the only stage that computes cost. +//! +//! Each Stage 2 candidate is checked against every query's accuracy target +//! with the accuracy model, and Count-Min is admitted only over weights proven +//! non-negative; a miss rejects the candidate as invalid, with a reason. +//! Every valid candidate is priced node by node over its physical DAG, so a +//! node shared by several queries is charged once, and the cheapest is +//! selected. The rest are reported valid but costlier. +//! +//! Prices come from [`crate::analytical_cost::estimate_operator`] over edge +//! statistics derived from the [`DataWorkload`] and a fixed default group +//! count; summary build and estimation are priced as rows × sketch depth and +//! groups × k. These numbers are illustrative, not calibrated. Latency +//! bounds and deployment capabilities are not checked yet. +use std::collections::{BTreeMap, HashMap}; + +use asap_types::ir::export::{ + NonASAPOpKind, PhysicalASAPDAG, PhysicalASAPNodeId, PhysicalASAPOperatorPayload as Payload, +}; +use asap_types::ir::{ASAPOp, Operator, OperatorNode}; +use asap_types::post_asap::{ + FieldDataType, SketchAlgorithm, SketchParams, SketchStatistic, WeightDomain, +}; +use asap_types::pre_asap::{DataType, Reduction, Schema}; +use asap_types::types::AccuracyTarget; +use asap_types::workload::DataWorkload; +use thiserror::Error; + +use crate::analytical_cost::{ + estimate_operator, AnalyticalCostError, PhysicalOperator, ResourceCalibration, ResourceEstimate, +}; +use crate::physical_candidates::PhysicalCandidate; +use crate::physical_operator_statistics::{ + EdgeStatistics, OperatorStatistics, PartitionStatistics, UnaryEdgeStatistics, +}; +use crate::PlanningModels; + +pub const COST_UNIT: &str = "cpu_ms_per_workload_evaluation"; +pub const COST_SOURCE: &str = "analytical-cost-v1 (illustrative statistics)"; + +/// Groups assumed for every `by (...)` reduction, absent group-count evidence. +const DEFAULT_GROUP_COUNT: u64 = 100; +/// Used only when the data workload does not declare them. +const DEFAULT_SERIES: u64 = 1_000; +const DEFAULT_ROWS_PER_SECOND: f64 = 1_000.0; +const DEFAULT_LOOKBACK_MS: u64 = 60_000; + +#[derive(Debug, Clone, PartialEq)] +pub struct NodeCost { + pub cost: f64, + pub detail: String, +} + +/// Whole-workload cost of one candidate: one entry per DAG node, `total` is +/// their sum. +#[derive(Debug, Clone, PartialEq)] +pub struct CandidateCost { + pub total: f64, + pub unit: &'static str, + pub source: &'static str, + pub per_node: BTreeMap, +} + +/// A candidate that was not selected. `valid == false`: it failed a check; +/// `true`: it lost on cost. +#[derive(Debug, Clone, PartialEq)] +pub struct Rejection { + pub id: String, + pub valid: bool, + pub reason: String, +} + +/// Costs are present for valid candidates only. +#[derive(Debug, Clone, PartialEq)] +pub struct Selection { + pub selected: String, + pub costs: BTreeMap, + pub rejected: Vec, +} + +#[derive(Debug, Error)] +pub enum SelectionError { + #[error("candidate {candidate} has {roots} roots but {targets} accuracy targets were given")] + TargetCount { + candidate: String, + roots: usize, + targets: usize, + }, + #[error("candidate {candidate}, node {node:?}: {error}")] + Cost { + candidate: String, + node: PhysicalASAPNodeId, + error: AnalyticalCostError, + }, + #[error("no valid candidate: {0:?}")] + NoValidCandidate(Vec), +} + +/// Reject candidates that miss a query's accuracy target, price the rest and +/// select the cheapest (the first on ties). `targets[i]` is the requirement +/// of `candidate.roots[i]`; `None` imposes none. +pub fn stage3_select( + cands: &[PhysicalCandidate], + targets: &[Option], + data: &DataWorkload, + models: PlanningModels<'_>, +) -> Result { + let mut costs = BTreeMap::new(); + let mut rejected = Vec::new(); + let mut best: Option<(&str, f64)> = None; + for candidate in cands { + if candidate.roots.len() != targets.len() { + return Err(SelectionError::TargetCount { + candidate: candidate.id.clone(), + roots: candidate.roots.len(), + targets: targets.len(), + }); + } + if let Some(reason) = accuracy_violation(candidate, targets, &models) { + rejected.push(Rejection { + id: candidate.id.clone(), + valid: false, + reason, + }); + continue; + } + let cost = price(&candidate.dag, data).map_err(|(node, error)| SelectionError::Cost { + candidate: candidate.id.clone(), + node, + error, + })?; + if best.is_none_or(|(_, total)| cost.total < total) { + best = Some((&candidate.id, cost.total)); + } + costs.insert(candidate.id.clone(), cost); + } + let Some((selected, best_total)) = best else { + return Err(SelectionError::NoValidCandidate(rejected)); + }; + for candidate in cands { + if candidate.id != selected && costs.contains_key(&candidate.id) { + rejected.push(Rejection { + id: candidate.id.clone(), + valid: true, + reason: format!( + "costlier: {:.3} vs {:.3} {COST_UNIT}", + costs[&candidate.id].total, best_total + ), + }); + } + } + Ok(Selection { + selected: selected.to_string(), + costs, + rejected, + }) +} + +/// The first summary estimate that misses its query's target, as a reason. +fn accuracy_violation( + candidate: &PhysicalCandidate, + targets: &[Option], + models: &PlanningModels<'_>, +) -> Option { + for (query, (root, target)) in candidate.roots.iter().zip(targets).enumerate() { + let Some(target) = target else { continue }; + for node in OperatorNode::reachable(root) { + let Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input, + query: statistic, + }) = &node.operator + else { + continue; + }; + let Operator::ASAP(ASAPOp::SummaryAgg { family, input, .. }) = &summary_input.operator + else { + return Some(format!("q{}: estimate over a non-summary input", query + 1)); + }; + let name = family_name(family); + // Count-Min's one-sided error bound assumes no negative updates. + if matches!(family, FieldDataType::Sketch(kind, _) + if matches!(kind.algorithm(), SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap)) + && !matches!(input.weight_domain, WeightDomain::NonNegative { .. }) + { + return Some(format!( + "q{}: {name} needs non-negative update weights, and these are not proven \ + non-negative", + query + 1 + )); + } + let Some(guarantee) = models.accuracy.local_guarantee(family, statistic) else { + return Some(format!( + "q{}: no accuracy model for {name}; target {target:?}", + query + 1 + )); + }; + if !models.accuracy.satisfies(&guarantee, target) { + return Some(format!( + "q{}: {name} guarantees bound {:?}, failure probability {:?}, which misses \ + target {target:?} (analytical guarantee; no accuracy evidence)", + query + 1, + guarantee.bound.evaluate(), + guarantee.failure_probability.evaluate(), + )); + } + } + } + None +} + +fn family_name(family: &FieldDataType) -> String { + match family { + FieldDataType::Sketch(kind, _) => format!("{:?}", kind.algorithm()), + FieldDataType::ExactAggregate(kind, _) => format!("exact {kind:?} accumulator"), + other => format!("{other:?}"), + } +} + +/// Statistics the analytical model needs, derived once per workload. +struct Shape { + series: u64, + rows_per_ms: f64, +} + +/// Price every node of `dag` once. Nodes are exported children first, so +/// each node's input statistics are known when it is reached. +fn price( + dag: &PhysicalASAPDAG, + data: &DataWorkload, +) -> Result { + let series = data + .input_cardinality + .value + .unwrap_or(DEFAULT_SERIES) + .max(1); + let shape = Shape { + series, + rows_per_ms: data + .ingestion_rate + .value + .map_or(DEFAULT_ROWS_PER_SECOND, |rate| rate.0) + / 1_000.0, + }; + let calibration = ResourceCalibration { + cost_per_cpu_op: 1e-6, + cost_per_scan_byte: 1e-7, + cost_per_retained_byte: 0.0, + version: "illustrative-v1".into(), + }; + let nodes: HashMap<_, _> = dag.nodes.iter().map(|n| (n.id, n)).collect(); + let mut output: HashMap = HashMap::new(); + let mut per_node = BTreeMap::new(); + for node in &dag.nodes { + let inputs: Vec<_> = dag + .edges + .iter() + .filter(|e| e.consumer == node.id) + .map(|e| output[&e.producer]) + .collect(); + let input = inputs + .first() + .copied() + .unwrap_or(EdgeStatistics { rows: 1, bytes: 1 }); + let width = row_bytes(&node.output_schema); + let edge = |rows: u64| EdgeStatistics { + rows, + bytes: rows * width, + }; + let unary = |output| UnaryEdgeStatistics { + input, + output, + promql: None, + }; + let groups = |reduction: &Reduction| { + match reduction { + Reduction::Reduce(keys) if keys.keys().is_empty() && !keys.is_without() => 1, + Reduction::Reduce(keys) if !keys.is_without() => DEFAULT_GROUP_COUNT, + _ => shape.series, + } + .min(input.rows.max(1)) + }; + let (out, estimate, detail) = match &node.payload { + Payload::Relational { operator } => match operator { + NonASAPOpKind::Scan { .. } => { + // A scan reads what its time range keeps. + let lookback = dag + .edges + .iter() + .filter(|e| e.producer == node.id) + .filter_map(|e| match &nodes[&e.consumer].payload { + Payload::Relational { + operator: NonASAPOpKind::TimeRange { range, .. }, + } => Some(range.as_millis() as u64), + _ => None, + }) + .max() + .unwrap_or(DEFAULT_LOOKBACK_MS); + let out = edge(((shape.rows_per_ms * lookback as f64).round() as u64).max(1)); + let estimate = estimate_operator( + PhysicalOperator::Scan, + OperatorStatistics::Scan { + edges: UnaryEdgeStatistics { + input: out, + output: out, + promql: None, + }, + source_read_bytes: out.bytes, + }, + ); + (out, estimate, format!("scan {} samples", out.rows)) + } + NonASAPOpKind::Aggregate { + reduction, + measures, + .. + } => { + let group_count = groups(reduction); + let keys = match reduction { + Reduction::Reduce(keys) => keys.keys().len() as u64, + _ => 1, + }; + let out = edge(group_count); + let estimate = estimate_operator( + PhysicalOperator::HashAggregate { + grouping_key_count: keys, + accumulator_count: measures.len().max(1) as u64, + }, + OperatorStatistics::HashAggregate { + edges: unary(out), + group_count, + key_bytes: 16 * keys, + accumulator_bytes_per_group: 8, + }, + ); + ( + out, + estimate, + format!( + "hash aggregate {} rows into {group_count} groups", + input.rows + ), + ) + } + NonASAPOpKind::Sort { keys, partition_by } => { + let partitions = if partition_by.keys().is_empty() { + 1 + } else { + DEFAULT_GROUP_COUNT.min(input.rows.max(1)) + }; + let estimate = estimate_operator( + PhysicalOperator::InMemoryComparisonSort { + ordering_key_count: keys.len() as u64, + partitioned: partitions > 1, + }, + OperatorStatistics::InMemoryComparisonSort { + edges: unary(input), + input_partitioning: split(input, partitions), + }, + ); + ( + input, + estimate, + format!("sort {} rows in {partitions} partitions", input.rows), + ) + } + NonASAPOpKind::Limit { + n, + offset, + partition_by, + } => { + let partitions = if partition_by.keys().is_empty() { + 1 + } else { + DEFAULT_GROUP_COUNT + }; + let limit = n.map_or(u64::MAX, |n| (n as u64).saturating_mul(partitions)); + let offset = (*offset as u64).saturating_mul(partitions); + let out = edge(input.rows.saturating_sub(offset).min(limit)); + let estimate = estimate_operator( + PhysicalOperator::Limit { limit, offset }, + OperatorStatistics::Limit { edges: unary(out) }, + ); + (out, estimate, format!("limit to {} rows", out.rows)) + } + other => { + let estimate = estimate_operator( + PhysicalOperator::PassThrough, + OperatorStatistics::PassThrough { + edges: unary(input), + }, + ); + let name = match other { + NonASAPOpKind::TimeRange { range, .. } => format!("time range {range:?}"), + _ => "operator".into(), + }; + (input, estimate, format!("{name}: pass {} rows", input.rows)) + } + }, + Payload::SummaryAgg { + family, reduction, .. + } => { + let group_count = groups(reduction); + let (depth, state_bytes) = summary_shape(family); + let out = EdgeStatistics { + rows: group_count, + bytes: group_count * state_bytes, + }; + let ops = input.rows as f64 * depth as f64; + ( + out, + Ok(ResourceEstimate::new(ops, out.bytes, 0)), + format!( + "build {} into {group_count} states: {} rows x depth {depth}", + family_name(family), + input.rows + ), + ) + } + Payload::SummaryEstimate { query } => { + let per_group = match query { + SketchStatistic::TopK { k } => *k as u64, + _ => 1, + }; + let out = edge(input.rows * per_group); + ( + out, + Ok(ResourceEstimate::new(out.rows as f64, 0, 0)), + format!("estimate {} groups x {per_group}", input.rows), + ) + } + Payload::FinalizeExactAccumulator => ( + edge(input.rows), + Ok(ResourceEstimate::new(input.rows as f64, 0, 0)), + format!("finalize {} accumulators", input.rows), + ), + _ => ( + edge(input.rows), + Ok(ResourceEstimate::new(input.rows as f64, 0, 0)), + format!("{} rows", input.rows), + ), + }; + let cost = estimate + .and_then(|estimate| estimate.calibrated_cost(&calibration)) + .map_err(|error| (node.id, error))?; + output.insert(node.id, out); + per_node.insert(node.id, NodeCost { cost, detail }); + } + Ok(CandidateCost { + total: per_node.values().map(|n| n.cost).sum(), + unit: COST_UNIT, + source: COST_SOURCE, + per_node, + }) +} + +/// `input` split as evenly as integers allow into `partitions` parts. +fn split(input: EdgeStatistics, partitions: u64) -> PartitionStatistics { + let part = |total: u64, i: u64| total / partitions + u64::from(i < total % partitions); + PartitionStatistics { + partitions: (0..partitions) + .map(|i| EdgeStatistics { + rows: part(input.rows, i), + bytes: part(input.bytes, i), + }) + .collect(), + } +} + +/// Plain values are 8 bytes, strings 16; summary columns are sized apart. +fn row_bytes(schema: &Schema) -> u64 { + schema + .fields + .iter() + .map(|f| match &f.dtype { + FieldDataType::Plain(DataType::Utf8) => 16, + _ => 8, + }) + .sum::() + .max(1) +} + +/// Update operations per input row and bytes per state. +fn summary_shape(family: &FieldDataType) -> (u64, u64) { + match family { + FieldDataType::Sketch(kind, _) => match kind.params() { + SketchParams::Cms { width, depth } | SketchParams::CountSketch { width, depth } => { + (u64::from(*depth), 8 * u64::from(*width) * u64::from(*depth)) + } + SketchParams::CmsWithHeap { + width, + depth, + heap_size, + } + | SketchParams::CountSketchWithHeap { + width, + depth, + heap_size, + } => ( + u64::from(*depth) + 1, + 8 * u64::from(*width) * u64::from(*depth) + 24 * u64::from(*heap_size), + ), + _ => (1, 1_024), + }, + _ => (1, 8), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::physical_candidates::stage2_physical; + use crate::test_support::lower_promql; + use asap_types::ir::QueryRoot; + use asap_types::workload::{Evidence, Rate}; + + fn data() -> DataWorkload { + DataWorkload { + ingestion_rate: Evidence { + value: Some(Rate(10_000.0)), + ..Default::default() + }, + input_cardinality: Evidence { + value: Some(10_000), + ..Default::default() + }, + ..Default::default() + } + } + + /// Exact (P1), CMS+heap (P2) and CountSketch+heap (P3) realizations of + /// one approximate top-k query, in Pass 1 catalog order. + fn candidates() -> Vec { + let root = lower_promql( + "topk by (job) (10, sum_over_time(m[1m]))", + AccuracyTarget::EpsilonDelta { + epsilon: 0.01, + delta: 0.001, + }, + ); + let inventory = crate::per_computation_candidates::generate_per_computation_candidates( + vec![(0, QueryRoot::Operator(root))], + ) + .unwrap(); + let topk = inventory + .targets + .iter() + .position(|t| t.candidates.len() > 2) + .unwrap(); + [0, 1, 2] + .into_iter() + .map(|alternative| { + let mut choice = vec![0; inventory.targets.len()]; + choice[topk] = alternative; + let roots: Vec<_> = crate::per_computation_candidates::compose_logical_candidate( + &inventory, &choice, + ) + .unwrap() + .into_iter() + .map(|(_, root)| match root { + QueryRoot::Operator(node) => node, + QueryRoot::Scalar(_) => panic!("operator root"), + }) + .collect(); + let mut candidate = stage2_physical("L", &roots).unwrap(); + candidate.id = format!("P{}", alternative + 1); + candidate + }) + .collect() + } + + /// A summary whose analytical guarantee misses the target is rejected as + /// invalid with a reason; the exact plan is then selected. + #[test] + fn accuracy_failing_candidate_is_rejected_as_invalid() { + let candidates = candidates(); + let strict = AccuracyTarget::EpsilonDelta { + epsilon: 1e-6, + delta: 1e-9, + }; + let selection = stage3_select( + &candidates, + &[Some(strict)], + &data(), + PlanningModels::builtin(), + ) + .unwrap(); + assert_eq!(selection.selected, "P1"); + let rejected: BTreeMap<_, _> = selection + .rejected + .iter() + .map(|r| (r.id.as_str(), r)) + .collect(); + assert_eq!(rejected.len(), 2); + assert!(!rejected["P3"].valid); + assert!( + rejected["P3"].reason.contains("misses target"), + "{}", + rejected["P3"].reason + ); + assert!(!selection.costs.contains_key("P3")); + } + + /// Count-Min over weights not proven non-negative is invalid whatever + /// the target. + #[test] + fn count_min_over_signed_weights_is_rejected_as_invalid() { + let target = AccuracyTarget::EpsilonDelta { + epsilon: 0.01, + delta: 0.001, + }; + let selection = stage3_select( + &candidates(), + &[Some(target)], + &data(), + PlanningModels::builtin(), + ) + .unwrap(); + let p2 = selection.rejected.iter().find(|r| r.id == "P2").unwrap(); + assert!(!p2.valid); + assert!(p2.reason.contains("non-negative"), "{}", p2.reason); + } + + /// Each node is priced once, under its DAG id, and the total is the sum; + /// every candidate is either selected or rejected as costlier. + #[test] + fn per_node_costs_cover_the_dag_and_sum_to_the_total() { + let candidates = candidates(); + let target = AccuracyTarget::EpsilonDelta { + epsilon: 0.01, + delta: 0.001, + }; + let selection = stage3_select( + &candidates, + &[Some(target)], + &data(), + PlanningModels::builtin(), + ) + .unwrap(); + for candidate in candidates.iter().filter(|c| c.id != "P2") { + let cost = &selection.costs[&candidate.id]; + let ids: Vec<_> = candidate.dag.nodes.iter().map(|n| n.id).collect(); + let mut keys: Vec<_> = cost.per_node.keys().copied().collect(); + keys.sort(); + let mut sorted = ids.clone(); + sorted.sort(); + assert_eq!(keys, sorted); + let sum: f64 = cost.per_node.values().map(|n| n.cost).sum(); + assert_eq!(cost.total, sum); + assert!(cost.total > 0.0); + } + let costlier: Vec<_> = selection.rejected.iter().filter(|r| r.valid).collect(); + assert_eq!(costlier.len(), 1); + assert_ne!(costlier[0].id, selection.selected); + } +} From d2924f4580a2ba968c3b4f4c9e0461415c161d0f Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sat, 3 Oct 2026 21:00:37 +0000 Subject: [PATCH 02/17] feat(devtools): stage_pipeline emits all four stages for Example 1 Add stage2_physical_asap (no cost) and stage3_selection (costs, selected, every other candidate rejected as invalid or costlier) per the v2 contract. PromQL rows carry the series identity from Stage 0, as runtime state needs. Regenerate tools/dag-viewer/examples/planner-layering-example1.json. Co-Authored-By: Claude Opus 5.5 --- crates/devtools/src/bin/stage_pipeline.rs | 93 ++++++++++++++++++++--- crates/devtools/tests/stage_pipeline.rs | 57 +++++++++++--- 2 files changed, 131 insertions(+), 19 deletions(-) diff --git a/crates/devtools/src/bin/stage_pipeline.rs b/crates/devtools/src/bin/stage_pipeline.rs index 23fe0d06..da02996c 100644 --- a/crates/devtools/src/bin/stage_pipeline.rs +++ b/crates/devtools/src/bin/stage_pipeline.rs @@ -4,15 +4,17 @@ // --promql "topk by (job) (10, rate(x[1m]))" --epsilon 0.01 --delta 0.001 --out run.json // // Writes an `asap-stage-pipeline/v1` document (tools/dag-viewer) with the -// first two planner stages only: +// four planner stages (#509 MVP): // - language_specific_frontends: the frontends' workload DAG, one root per query; // - logical_asap_optimization: Pass 1 whole-workload candidates, one per // choice of a per-computation candidate for every target (the Cartesian // product), in enumeration order and capped by `--max-candidates` // (default 64). A candidate that cannot be composed carries `error` -// instead of `dag`. -// Physical ASAP-aware optimization and plan selection, and therefore cost, -// are absent. +// instead of `dag`; +// - stage2_physical_asap: one physical candidate per composed logical +// candidate (operator implementation only, everything at query time), no cost; +// - stage3_selection: per-candidate costs, the selected candidate, and +// every other candidate as rejected (`valid: false`) or costlier. // // `--promql` may repeat. `--epsilon`/`--delta` apply to every `--promql` // query; without them the queries are exact. `--interval-ms` is the source @@ -24,7 +26,11 @@ use asap_aware_mapping::per_computation_candidates::{ compose_logical_candidate, generate_per_computation_candidates, ComputationCandidate, PerComputationCandidates, }; +use asap_aware_mapping::physical_candidates::stage2_physical; +use asap_aware_mapping::plan_selection::{stage3_select, Selection}; +use asap_aware_mapping::PlanningModels; use asap_types::ir::flat::{flatten, FlatDag}; +use asap_types::ir::schema_support::with_promql_series_identity; use asap_types::ir::{OperatorNode, QueryRoot}; use asap_types::post_asap::SketchAlgorithm; use asap_types::types::AccuracyTarget; @@ -88,8 +94,18 @@ fn run(args: Vec) -> Result<(), String> { } fn stage_pipeline(workload: &PlanningWorkload, max_candidates: usize) -> Result { + // PromQL rows carry each series' full identity as a column: the row + // representation per-series state needs at runtime. let roots = asap_frontend_promql::lower_promql_query_workload(workload, 0) - .map_err(|e| format!("lowering: {e}"))?; + .map_err(|e| format!("lowering: {e}"))? + .into_iter() + .map(|root| match root { + QueryRoot::Operator(node) => with_promql_series_identity(&node) + .map(QueryRoot::Operator) + .map_err(|e| format!("series identity: {e}")), + scalar => Ok(scalar), + }) + .collect::, _>>()?; let frontends_dag = export(&roots)?; let inventory = generate_per_computation_candidates(roots.into_iter().enumerate().collect()) .map_err(|e| format!("Pass 1: {e}"))?; @@ -100,18 +116,34 @@ fn stage_pipeline(workload: &PlanningWorkload, max_candidates: usize) -> Result< .product::(); let owners = target_owners(&inventory); let mut candidates = Vec::new(); + let mut physical = Vec::new(); let mut choice = vec![0; inventory.targets.len()]; for index in 0..combinations.min(max_candidates) { - let mut candidate = json!({ - "id": format!("L{}", index + 1), - "label": label(&inventory, &owners, &choice), - }); + let (id, label) = ( + format!("L{}", index + 1), + label(&inventory, &owners, &choice), + ); + let mut candidate = json!({ "id": id, "label": label }); // A choice that composition cannot substitute yet is listed with its // error rather than ending the run. match compose_logical_candidate(&inventory, &choice) { Ok(roots) => { let roots: Vec<_> = roots.into_iter().map(|(_, root)| root).collect(); candidate["dag"] = json!(export(&roots)?); + let operators = roots + .into_iter() + .map(|root| match root { + QueryRoot::Operator(node) => Ok(node), + QueryRoot::Scalar(_) => { + Err("Stage 2: scalar query roots are not physical yet") + } + }) + .collect::, _>>()?; + let mut physical_candidate = + stage2_physical(&id, &operators).map_err(|e| format!("Stage 2 {id}: {e}"))?; + physical_candidate.id = format!("P{}", index + 1); + physical_candidate.label = label; + physical.push(physical_candidate); } Err(error) => candidate["error"] = json!(error.to_string()), } @@ -125,6 +157,18 @@ fn stage_pipeline(workload: &PlanningWorkload, max_candidates: usize) -> Result< *digit = 0; } } + let targets: Vec<_> = workload + .query_workload + .entries() + .map(|entry| Some(entry.requirements.accuracy.target())) + .collect(); + let data = workload.data_workload.clone().unwrap_or_default(); + let selection = stage3_select(&physical, &targets, &data, PlanningModels::builtin()) + .map_err(|e| format!("Stage 3: {e}"))?; + let stage2: Vec<_> = physical + .iter() + .map(|p| json!({ "id": p.id, "from_logical": p.from_logical, "label": p.label, "dag": p.dag })) + .collect(); Ok(json!({ "format": "asap-stage-pipeline/v1", "workload": { "queries": workload_queries(workload) }, @@ -134,9 +178,40 @@ fn stage_pipeline(workload: &PlanningWorkload, max_candidates: usize) -> Result< "capped": combinations > max_candidates, "candidates": candidates, }, + "stage2_physical_asap": { "candidates": stage2 }, + "stage3_selection": stage3_json(&selection), })) } +fn stage3_json(selection: &Selection) -> Value { + let costs: serde_json::Map<_, _> = selection + .costs + .iter() + .map(|(id, cost)| { + let per_node: serde_json::Map<_, _> = cost + .per_node + .iter() + .map(|(node, c)| { + ( + node.0.to_string(), + json!({ "cost": c.cost, "detail": c.detail }), + ) + }) + .collect(); + ( + id.clone(), + json!({ "total": cost.total, "unit": cost.unit, "source": cost.source, "per_node": per_node }), + ) + }) + .collect(); + let rejected: Vec<_> = selection + .rejected + .iter() + .map(|r| json!({ "id": r.id, "valid": r.valid, "reason": r.reason })) + .collect(); + json!({ "costs": costs, "selected": selection.selected, "rejected": rejected }) +} + /// A logical DAG as a flat node list: node `i` is `nodes[i]`, and children are /// node indices inside each operator. fn export(roots: &[QueryRoot]) -> Result { diff --git a/crates/devtools/tests/stage_pipeline.rs b/crates/devtools/tests/stage_pipeline.rs index ef5a688b..fbd56e23 100644 --- a/crates/devtools/tests/stage_pipeline.rs +++ b/crates/devtools/tests/stage_pipeline.rs @@ -1,11 +1,10 @@ -//! `stage_pipeline` writes a valid viewer document for #509 Example 1: the -//! language-specific frontends' DAG and the logical ASAP-aware optimization -//! candidates. +//! `stage_pipeline` writes a valid four-stage viewer document for #509 Example 1. use std::collections::HashSet; use std::path::PathBuf; use std::process::Command; use asap_types::ir::flat::FlatDag; +use asap_types::ir::physical_export::{PhysicalASAPDAG, PhysicalASAPDAGDocument}; use asap_types::ir::QueryRoot; use serde_json::Value; @@ -47,9 +46,11 @@ fn validated_dag(value: &Value, queries: usize) { } } -/// Every DAG is a well-formed flat DAG with one root per query; ids are unique; -/// only the frontends and logical ASAP-aware optimization are present; the -/// committed fixture is current. +/// Every logical DAG is a well-formed flat DAG and every physical DAG validates; +/// each has one root per query; ids are unique; +/// Stage 2 maps the composed logical candidates one-to-one with no cost; +/// Stage 3 accounts for every Stage 2 candidate once and costs exactly the +/// valid ones; the committed fixture is current. #[test] fn example1_document_is_valid_and_committed_fixture_is_current() { let document = generate(&[]); @@ -63,8 +64,6 @@ fn example1_document_is_valid_and_committed_fixture_is_current() { assert_eq!(document["format"], "asap-stage-pipeline/v1"); let queries = document["workload"]["queries"].as_array().unwrap().len(); assert_eq!(queries, 2); - assert!(document.get("physical_asap_optimization").is_none()); - assert!(document.get("plan_selection").is_none()); validated_dag(&document["language_specific_frontends"]["dag"], queries); let logical = &document["logical_asap_optimization"]; @@ -72,10 +71,48 @@ fn example1_document_is_valid_and_committed_fixture_is_current() { assert_eq!(logical["capped"], false); assert_eq!(logical["combinations"], candidates.len()); let mut ids = HashSet::new(); + let mut composed = HashSet::new(); for candidate in candidates { - assert!(ids.insert(candidate["id"].as_str().unwrap())); - validated_dag(&candidate["dag"], queries); + let id = candidate["id"].as_str().unwrap(); + assert!(ids.insert(id)); + if candidate.get("dag").is_some() { + composed.insert(id); + validated_dag(&candidate["dag"], queries); + } + } + + let physical = document["stage2_physical_asap"]["candidates"] + .as_array() + .unwrap(); + let sources: HashSet<_> = physical + .iter() + .map(|p| p["from_logical"].as_str().unwrap()) + .collect(); + assert_eq!(physical.len(), composed.len()); + assert_eq!(sources, composed); + for candidate in physical { + assert!(candidate.get("cost").is_none(), "Stage 2 has no cost"); + let dag: PhysicalASAPDAG = serde_json::from_value(candidate["dag"].clone()).unwrap(); + assert_eq!(dag.roots.len(), queries); + PhysicalASAPDAGDocument::new(dag).validate().unwrap(); + } + + let stage3 = &document["stage3_selection"]; + let costs = stage3["costs"].as_object().unwrap(); + let selected = stage3["selected"].as_str().unwrap(); + let mut accounted = HashSet::from([selected]); + for rejection in stage3["rejected"].as_array().unwrap() { + let id = rejection["id"].as_str().unwrap(); + assert!(accounted.insert(id), "{id} accounted for once"); + assert!(!rejection["reason"].as_str().unwrap().is_empty()); + assert_eq!( + rejection["valid"].as_bool().unwrap(), + costs.contains_key(id) + ); } + assert!(costs.contains_key(selected)); + let all: HashSet<_> = physical.iter().map(|p| p["id"].as_str().unwrap()).collect(); + assert_eq!(accounted, all); } /// The Cartesian product is cut at `--max-candidates` and says so. From 616ad8a9ed60e7b7101a00bf189e70c00db10d2f Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Thu, 8 Oct 2026 14:04:07 +0000 Subject: [PATCH 03/17] refactor(planner): read Operator payloads in Stage 2/3 after the export removal Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W7qG9aFyPij5uWsyAJCxDW --- .../src/physical_candidates.rs | 22 ++++++------ .../asap-aware-mapping/src/plan_selection.rs | 34 +++++++++++-------- crates/devtools/src/bin/stage_pipeline.rs | 2 +- 3 files changed, 30 insertions(+), 28 deletions(-) diff --git a/crates/asap-aware-mapping/src/physical_candidates.rs b/crates/asap-aware-mapping/src/physical_candidates.rs index a581e4e0..5c0f70f6 100644 --- a/crates/asap-aware-mapping/src/physical_candidates.rs +++ b/crates/asap-aware-mapping/src/physical_candidates.rs @@ -11,7 +11,9 @@ use std::collections::HashMap; use std::rc::Rc; -use asap_types::ir::export::{compile_physical_asap_workload_with_node_ids, PhysicalASAPDAG}; +use asap_types::ir::physical_export::{ + compile_physical_asap_workload_with_node_ids, PhysicalASAPDAG, +}; use asap_types::ir::{ apply_materialization_timings, ASAPOp, MaterializationAssignment, NonASAPOp, Operator, OperatorNode, ScalarExpr, SchemaDerivationError, SortKey, TimingMemo, @@ -144,7 +146,7 @@ fn rebuilt(node: &Rc, memo: &Memo) -> Result, Sta .iter() .any(|child| !Rc::ptr_eq(child, &memo[&Rc::as_ptr(child)])); Ok(if changed { - Rc::new(node.map_children(|child| memo[&Rc::as_ptr(child)].clone())?) + Rc::new(node.with_new_children(|child| memo[&Rc::as_ptr(child)].clone())?) } else { node.clone() }) @@ -154,8 +156,8 @@ fn rebuilt(node: &Rc, memo: &Memo) -> Result, Sta mod tests { use super::*; use crate::test_support::lower_promql; - use asap_types::ir::export::PhysicalASAPOperatorPayload as Payload; - use asap_types::ir::export::{NonASAPOpKind, PhysicalASAPNodeId}; + use asap_types::ir::physical_export::PhysicalASAPNodeId; + use asap_types::ir::physical_export::PhysicalASAPOperatorPayload as Payload; use asap_types::ir::QueryRoot; use asap_types::post_asap::ExecutionTiming; use asap_types::types::AccuracyTarget; @@ -171,7 +173,7 @@ mod tests { let NonASAPOp::Aggregate { child: shared, .. } = topk.expect_non_asap() else { panic!("topk aggregate") }; - let sum = Rc::new(sum.map_children(|_| shared.clone()).unwrap()); + let sum = Rc::new(sum.with_new_children(|_| shared.clone()).unwrap()); let candidate = stage2_physical("L1", &[topk, sum]).unwrap(); let NonASAPOp::Limit { @@ -209,12 +211,8 @@ mod tests { .unwrap() .payload { - Payload::Relational { - operator: NonASAPOpKind::Sort { .. }, - } => "sort", - Payload::Relational { - operator: NonASAPOpKind::Limit { .. }, - } => "limit", + Payload::NonASAP(NonASAPOp::Sort { .. }) => "sort", + Payload::NonASAP(NonASAPOp::Limit { .. }) => "limit", _ => "other", }; let sort_id = dag @@ -268,7 +266,7 @@ mod tests { .dag .nodes .iter() - .any(|n| matches!(n.payload, Payload::SummaryAgg { .. }))); + .any(|n| matches!(n.payload, Payload::ASAP(ASAPOp::SummaryAgg { .. })))); assert!(candidate .dag .nodes diff --git a/crates/asap-aware-mapping/src/plan_selection.rs b/crates/asap-aware-mapping/src/plan_selection.rs index 9909a506..2067a4ee 100644 --- a/crates/asap-aware-mapping/src/plan_selection.rs +++ b/crates/asap-aware-mapping/src/plan_selection.rs @@ -12,10 +12,11 @@ //! count; summary build and estimation are priced as rows × sketch depth and //! groups × k. These numbers are illustrative, not calibrated. Latency //! bounds and deployment capabilities are not checked yet. +use asap_types::ir::NonASAPOp; use std::collections::{BTreeMap, HashMap}; -use asap_types::ir::export::{ - NonASAPOpKind, PhysicalASAPDAG, PhysicalASAPNodeId, PhysicalASAPOperatorPayload as Payload, +use asap_types::ir::physical_export::{ + PhysicalASAPDAG, PhysicalASAPNodeId, PhysicalASAPOperatorPayload as Payload, }; use asap_types::ir::{ASAPOp, Operator, OperatorNode}; use asap_types::post_asap::{ @@ -280,17 +281,17 @@ fn price( .min(input.rows.max(1)) }; let (out, estimate, detail) = match &node.payload { - Payload::Relational { operator } => match operator { - NonASAPOpKind::Scan { .. } => { + Payload::NonASAP(operator) => match operator { + NonASAPOp::Scan { .. } => { // A scan reads what its time range keeps. let lookback = dag .edges .iter() .filter(|e| e.producer == node.id) .filter_map(|e| match &nodes[&e.consumer].payload { - Payload::Relational { - operator: NonASAPOpKind::TimeRange { range, .. }, - } => Some(range.as_millis() as u64), + Payload::NonASAP(NonASAPOp::TimeRange { range, .. }) => { + Some(range.as_millis() as u64) + } _ => None, }) .max() @@ -309,7 +310,7 @@ fn price( ); (out, estimate, format!("scan {} samples", out.rows)) } - NonASAPOpKind::Aggregate { + NonASAPOp::Aggregate { reduction, measures, .. @@ -341,7 +342,9 @@ fn price( ), ) } - NonASAPOpKind::Sort { keys, partition_by } => { + NonASAPOp::Sort { + keys, partition_by, .. + } => { let partitions = if partition_by.keys().is_empty() { 1 } else { @@ -363,10 +366,11 @@ fn price( format!("sort {} rows in {partitions} partitions", input.rows), ) } - NonASAPOpKind::Limit { + NonASAPOp::Limit { n, offset, partition_by, + .. } => { let partitions = if partition_by.keys().is_empty() { 1 @@ -390,15 +394,15 @@ fn price( }, ); let name = match other { - NonASAPOpKind::TimeRange { range, .. } => format!("time range {range:?}"), + NonASAPOp::TimeRange { range, .. } => format!("time range {range:?}"), _ => "operator".into(), }; (input, estimate, format!("{name}: pass {} rows", input.rows)) } }, - Payload::SummaryAgg { + Payload::ASAP(ASAPOp::SummaryAgg { family, reduction, .. - } => { + }) => { let group_count = groups(reduction); let (depth, state_bytes) = summary_shape(family); let out = EdgeStatistics { @@ -416,7 +420,7 @@ fn price( ), ) } - Payload::SummaryEstimate { query } => { + Payload::ASAP(ASAPOp::SummaryEstimate { query, .. }) => { let per_group = match query { SketchStatistic::TopK { k } => *k as u64, _ => 1, @@ -428,7 +432,7 @@ fn price( format!("estimate {} groups x {per_group}", input.rows), ) } - Payload::FinalizeExactAccumulator => ( + Payload::ASAP(ASAPOp::FinalizeExactAccumulator { .. }) => ( edge(input.rows), Ok(ResourceEstimate::new(input.rows as f64, 0, 0)), format!("finalize {} accumulators", input.rows), diff --git a/crates/devtools/src/bin/stage_pipeline.rs b/crates/devtools/src/bin/stage_pipeline.rs index da02996c..3ebc7f31 100644 --- a/crates/devtools/src/bin/stage_pipeline.rs +++ b/crates/devtools/src/bin/stage_pipeline.rs @@ -193,7 +193,7 @@ fn stage3_json(selection: &Selection) -> Value { .iter() .map(|(node, c)| { ( - node.0.to_string(), + node.to_string(), json!({ "cost": c.cost, "detail": c.detail }), ) }) From e6669ed43a9b45b8923927e50719f85b91b0c8fe Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 13 Oct 2026 03:10:02 +0000 Subject: [PATCH 04/17] fix(planner): Stage 2/3 on #575's composed candidates, query roots and #667 materialization Stage 2 takes query roots, scalar ones included (#542), times every sub-DAG with Materialization::NotMaterialized (#667) and keeps the guarantee Stage 1 composed on every node it implements; a MaintainPopulation is no longer rejected, since it now runs at query time when not materialized. Stage 3 accepts a query's output when its root guarantee meets the target, or when the deployment's accuracy model shows this candidate's output does (AccuracyModel::output_guarantee, none by default). A certified estimate inside the plan is not enough: a difference of two quantile estimates stays listed as invalid, with a reason, under the built-in models. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W7qG9aFyPij5uWsyAJCxDW --- crates/asap-aware-mapping/src/accuracy/mod.rs | 12 +- .../src/physical_candidates.rs | 148 ++++++----- .../asap-aware-mapping/src/plan_selection.rs | 245 +++++++++++++++--- crates/devtools/src/bin/stage_pipeline.rs | 13 +- 4 files changed, 299 insertions(+), 119 deletions(-) diff --git a/crates/asap-aware-mapping/src/accuracy/mod.rs b/crates/asap-aware-mapping/src/accuracy/mod.rs index a2f3354c..1c6d650c 100644 --- a/crates/asap-aware-mapping/src/accuracy/mod.rs +++ b/crates/asap-aware-mapping/src/accuracy/mod.rs @@ -20,7 +20,7 @@ pub use evidence::{ QuantileInputDomain, WorkloadAccuracyEvidence, }; -use asap_types::ir::OperatorNode; +use asap_types::ir::{OperatorNode, QueryRoot}; use asap_types::post_asap::{ AccuracyError, BoundExpr, CompositionOperator, ErrorMetric, FieldDataType, GuaranteeSource, ProbabilityExpr, ResultGuarantee, SketchAlgorithm, SketchParams, SketchStatistic, @@ -65,6 +65,16 @@ pub trait AccuracyModel { stats: &PropagationStats, ) -> Result; + /// The deployment's empirical accuracy for one candidate's whole query + /// output, as a guarantee. Plan selection (#509 Stage 3) accepts a + /// candidate whose output has no theoretical guarantee that meets the + /// target, e.g. a difference of two quantile estimates, only when this + /// shows the output meets it. The built-in model has no empirical + /// evidence. + fn output_guarantee(&self, _output: &QueryRoot) -> Option { + None + } + /// Compare the dimensions requested by `target`. Unknown required /// dimensions fail; selection separately excludes missing accuracy evidence. fn satisfies(&self, guarantee: &ResultGuarantee, target: &AccuracyTarget) -> bool; diff --git a/crates/asap-aware-mapping/src/physical_candidates.rs b/crates/asap-aware-mapping/src/physical_candidates.rs index 5c0f70f6..d35451f4 100644 --- a/crates/asap-aware-mapping/src/physical_candidates.rs +++ b/crates/asap-aware-mapping/src/physical_candidates.rs @@ -1,13 +1,16 @@ //! #509 Stage 2 (MVP): physical operator implementation of one logical -//! candidate. No materialization choice is made: every node runs at query -//! time, so each logical candidate yields exactly one physical candidate. +//! candidate. Materialization offers one option for now: every sub-DAG is +//! `Materialization::NotMaterialized` and runs at query time, so each +//! logical candidate yields exactly one physical candidate. Enumerating +//! `MatAtIngestion(medium)` and `MatAtQuery(medium)` (Algorithm 3) comes +//! later. //! //! The runtime has one implementation per logical operator except exact //! top-k, which it cannot run as `Aggregate{[TopK]}`. Stage 2 records that //! choice in the DAG by rewriting it to a per-group sort followed by a //! per-group limit, the shape the PromQL frontend uses for generic `topk`. //! A summary needs no rewrite: `SummaryAgg` → `SummaryEstimate` already is -//! build → estimate. +//! build → estimate. Every node keeps the guarantee Stage 1 composed for it. use std::collections::HashMap; use std::rc::Rc; @@ -15,24 +18,24 @@ use asap_types::ir::physical_export::{ compile_physical_asap_workload_with_node_ids, PhysicalASAPDAG, }; use asap_types::ir::{ - apply_materialization_timings, ASAPOp, MaterializationAssignment, NonASAPOp, Operator, - OperatorNode, ScalarExpr, SchemaDerivationError, SortKey, TimingMemo, + apply_materialization_timings, MaterializationAssignment, NonASAPOp, Operator, OperatorNode, + QueryRoot, ScalarExpr, SchemaDerivationError, SortKey, TimingMemo, }; -use asap_types::post_asap::ExecutionDataStateError; +use asap_types::post_asap::{ExecutionDataStateError, Materialization}; use asap_types::pre_asap::column_resolution::resolve_column_ref; use asap_types::pre_asap::expr_ir::ColumnRef; use asap_types::pre_asap::{AggIntent, Reduction}; use thiserror::Error; /// One Stage 2 candidate, derived from exactly one Stage 1 candidate. -/// `roots` are the timed operator roots (one per query, in workload order) +/// `roots` are the timed query roots (one per query, in workload order) /// that `dag` exports. #[derive(Debug, Clone)] pub struct PhysicalCandidate { pub id: String, pub from_logical: String, pub label: String, - pub roots: Vec>, + pub roots: Vec, pub dag: PhysicalASAPDAG, } @@ -44,30 +47,27 @@ pub enum Stage2Error { Timing(#[from] ExecutionDataStateError), #[error("exact top-k has no sortable value column: {0}")] NoValueColumn(String), - #[error("maintained populations run at ingestion time, which Stage 2 does not plan yet")] - IngestionTimeOnly, } -/// Implement every operator of one logical candidate and time it at query -/// time. `id` and `label` are left empty for the caller to name. Sharing -/// between `roots` is preserved: one memo serves the whole workload. +/// Implement every operator of one logical candidate and time it with every +/// sub-DAG `NotMaterialized`. `id` and `label` are left empty for the caller +/// to name. Sharing between `roots` is preserved: one memo serves the whole +/// workload. pub fn stage2_physical( from_logical: &str, - roots: &[Rc], + roots: &[QueryRoot], ) -> Result { let mut memo = HashMap::new(); - let implemented = roots - .iter() - .map(|root| implement(root, &mut memo)) - .collect::, _>>()?; - reject_maintained_populations(&implemented)?; - // The default assignment computes every summary state at query time. - let assignment = MaterializationAssignment::default(); + let implemented = map_roots(roots, &mut |node| implement(node, &mut memo))?; + let assignment = MaterializationAssignment::with_default(Materialization::NotMaterialized); let mut timing = TimingMemo::new(); - let timed = implemented - .iter() - .map(|root| apply_materialization_timings(root, &assignment, &mut timing)) - .collect::, _>>()?; + let timed = map_roots(&implemented, &mut |node| { + Ok(apply_materialization_timings( + node, + &assignment, + &mut timing, + )?) + })?; let dag = compile_physical_asap_workload_with_node_ids(&timed)?.dag; Ok(PhysicalCandidate { id: String::new(), @@ -78,19 +78,26 @@ pub fn stage2_physical( }) } -/// A maintained population always runs at ingestion time, which Stage 2 -/// does not plan yet. -fn reject_maintained_populations(roots: &[Rc]) -> Result<(), Stage2Error> { - let maintained = roots.iter().flat_map(OperatorNode::reachable).any(|node| { - matches!( - node.operator, - Operator::ASAP(ASAPOp::MaintainPopulation { .. }) - ) - }); - if maintained { - return Err(Stage2Error::IngestionTimeOnly); - } - Ok(()) +/// `roots` with every operator node they read replaced by `f`'s result. +fn map_roots( + roots: &[QueryRoot], + f: &mut impl FnMut(&Rc) -> Result, Stage2Error>, +) -> Result, Stage2Error> { + roots + .iter() + .map(|root| match root { + QueryRoot::Operator(node) => Ok(QueryRoot::Operator(f(node)?)), + QueryRoot::Scalar(expr) => { + let mut mapped = HashMap::new(); + for node in expr.operator_refs() { + mapped.insert(Rc::as_ptr(node), f(node)?); + } + Ok(QueryRoot::Scalar(expr.map_operator_refs(&mut |node| { + mapped[&Rc::as_ptr(node)].clone() + }))) + } + }) + .collect() } type Memo = HashMap<*const OperatorNode, Rc>; @@ -115,21 +122,28 @@ fn implement(node: &Rc, memo: &mut Memo) -> Result rebuilt(node, memo)?, }, @@ -146,7 +160,10 @@ fn rebuilt(node: &Rc, memo: &Memo) -> Result, Sta .iter() .any(|child| !Rc::ptr_eq(child, &memo[&Rc::as_ptr(child)])); Ok(if changed { - Rc::new(node.with_new_children(|child| memo[&Rc::as_ptr(child)].clone())?) + Rc::new( + node.with_new_children(|child| memo[&Rc::as_ptr(child)].clone())? + .with_guarantee(node.guarantee.clone()), + ) } else { node.clone() }) @@ -158,7 +175,7 @@ mod tests { use crate::test_support::lower_promql; use asap_types::ir::physical_export::PhysicalASAPNodeId; use asap_types::ir::physical_export::PhysicalASAPOperatorPayload as Payload; - use asap_types::ir::QueryRoot; + use asap_types::ir::ASAPOp; use asap_types::post_asap::ExecutionTiming; use asap_types::types::AccuracyTarget; @@ -174,14 +191,18 @@ mod tests { panic!("topk aggregate") }; let sum = Rc::new(sum.with_new_children(|_| shared.clone()).unwrap()); - let candidate = stage2_physical("L1", &[topk, sum]).unwrap(); + let candidate = stage2_physical("L1", &[topk.into(), sum.into()]).unwrap(); + let root = |i: usize| match &candidate.roots[i] { + QueryRoot::Operator(node) => node.clone(), + QueryRoot::Scalar(_) => panic!("operator root"), + }; let NonASAPOp::Limit { n: Some(10), partition_by, child: sort, .. - } = candidate.roots[0].expect_non_asap() + } = root(0).expect_non_asap().clone() else { panic!("limit root") }; @@ -194,14 +215,14 @@ mod tests { else { panic!("sort below limit") }; - assert_eq!(sort_partition, partition_by); + assert_eq!(*sort_partition, partition_by); let NonASAPOp::Aggregate { child: below_sum, .. - } = candidate.roots[1].expect_non_asap() + } = root(1).expect_non_asap().clone() else { panic!("sum root") }; - assert!(Rc::ptr_eq(below_sort, below_sum)); + assert!(Rc::ptr_eq(below_sort, &below_sum)); let dag = &candidate.dag; let kind = |id: PhysicalASAPNodeId| match &dag @@ -250,16 +271,13 @@ mod tests { let choice: Vec<_> = inventory .targets .iter() - .map(|t| t.candidates.len() - 1) + .map(|t| t.choices.candidates.len() - 1) .collect(); let roots: Vec<_> = crate::per_computation_candidates::compose_logical_candidate(&inventory, &choice) .unwrap() .into_iter() - .map(|(_, root)| match root { - QueryRoot::Operator(node) => node, - QueryRoot::Scalar(_) => panic!("operator root"), - }) + .map(|(_, root)| root) .collect(); let candidate = stage2_physical("L1", &roots).unwrap(); assert!(candidate diff --git a/crates/asap-aware-mapping/src/plan_selection.rs b/crates/asap-aware-mapping/src/plan_selection.rs index 2067a4ee..1b798240 100644 --- a/crates/asap-aware-mapping/src/plan_selection.rs +++ b/crates/asap-aware-mapping/src/plan_selection.rs @@ -1,8 +1,11 @@ //! #509 Stage 3 (MVP): plan selection, the only stage that computes cost. //! -//! Each Stage 2 candidate is checked against every query's accuracy target -//! with the accuracy model, and Count-Min is admitted only over weights proven -//! non-negative; a miss rejects the candidate as invalid, with a reason. +//! Each Stage 2 candidate is checked against every query's accuracy target: +//! the query's output meets it when its root guarantee (composed in Stage 1) +//! does, or when the deployment's accuracy model shows this candidate's +//! output does. Count-Min is admitted only over weights proven non-negative. +//! A miss rejects the candidate as invalid, with a reason; only this stage +//! discards valid candidates. //! Every valid candidate is priced node by node over its physical DAG, so a //! node shared by several queries is charged once, and the cheapest is //! selected. The rest are reported valid but costlier. @@ -12,15 +15,15 @@ //! count; summary build and estimation are priced as rows × sketch depth and //! groups × k. These numbers are illustrative, not calibrated. Latency //! bounds and deployment capabilities are not checked yet. -use asap_types::ir::NonASAPOp; use std::collections::{BTreeMap, HashMap}; +use std::rc::Rc; use asap_types::ir::physical_export::{ PhysicalASAPDAG, PhysicalASAPNodeId, PhysicalASAPOperatorPayload as Payload, }; -use asap_types::ir::{ASAPOp, Operator, OperatorNode}; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, QueryRoot}; use asap_types::post_asap::{ - FieldDataType, SketchAlgorithm, SketchParams, SketchStatistic, WeightDomain, + FieldDataType, ResultGuarantee, SketchAlgorithm, SketchParams, SketchStatistic, WeightDomain, }; use asap_types::pre_asap::{DataType, Reduction, Schema}; use asap_types::types::AccuracyTarget; @@ -157,58 +160,92 @@ pub fn stage3_select( }) } -/// The first summary estimate that misses its query's target, as a reason. +/// The first query whose output is not shown to meet its target, as a +/// reason. An output meets its target when its root guarantee (composed in +/// Stage 1) satisfies it, or when the deployment's accuracy model shows that +/// this candidate's output does ([`crate::accuracy::AccuracyModel::output_guarantee`]). A +/// certified estimate inside the plan is not enough on its own: a difference +/// or ratio of certified estimates can be unbounded. Count-Min is admitted +/// only over update weights proven non-negative, which its bound assumes. fn accuracy_violation( candidate: &PhysicalCandidate, targets: &[Option], models: &PlanningModels<'_>, ) -> Option { for (query, (root, target)) in candidate.roots.iter().zip(targets).enumerate() { + let query = query + 1; let Some(target) = target else { continue }; - for node in OperatorNode::reachable(root) { - let Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input, - query: statistic, - }) = &node.operator - else { + for node in root_operators(root) + .iter() + .flat_map(|node| OperatorNode::reachable(node)) + { + let Operator::ASAP(ASAPOp::SummaryAgg { family, input, .. }) = &node.operator else { continue; }; - let Operator::ASAP(ASAPOp::SummaryAgg { family, input, .. }) = &summary_input.operator - else { - return Some(format!("q{}: estimate over a non-summary input", query + 1)); - }; - let name = family_name(family); - // Count-Min's one-sided error bound assumes no negative updates. if matches!(family, FieldDataType::Sketch(kind, _) if matches!(kind.algorithm(), SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap)) && !matches!(input.weight_domain, WeightDomain::NonNegative { .. }) { return Some(format!( - "q{}: {name} needs non-negative update weights, and these are not proven \ + "q{query}: {} needs non-negative update weights, and these are not proven \ non-negative", - query + 1 - )); - } - let Some(guarantee) = models.accuracy.local_guarantee(family, statistic) else { - return Some(format!( - "q{}: no accuracy model for {name}; target {target:?}", - query + 1 - )); - }; - if !models.accuracy.satisfies(&guarantee, target) { - return Some(format!( - "q{}: {name} guarantees bound {:?}, failure probability {:?}, which misses \ - target {target:?} (analytical guarantee; no accuracy evidence)", - query + 1, - guarantee.bound.evaluate(), - guarantee.failure_probability.evaluate(), + family_name(family) )); } } + let guarantee = root_guarantee(root); + let meets = |g: &ResultGuarantee| models.accuracy.satisfies(g, target); + if guarantee.as_ref().is_some_and(meets) + || models + .accuracy + .output_guarantee(root) + .as_ref() + .is_some_and(meets) + { + continue; + } + return Some(match guarantee { + Some(g) => format!( + "q{query}: the output guarantees bound {:?}, failure probability {:?}, which \ + misses target {target:?}, and the accuracy model has no evidence that it meets it", + g.bound.evaluate(), + g.failure_probability.evaluate(), + ), + None => format!( + "q{query}: the output has no guarantee, and the accuracy model has no evidence \ + that it meets target {target:?}" + ), + }); } None } +/// The operator nodes a query root reads. +fn root_operators(root: &QueryRoot) -> Vec<&Rc> { + match root { + QueryRoot::Operator(node) => vec![node], + QueryRoot::Scalar(expr) => expr.operator_refs(), + } +} + +/// The theoretical guarantee of a query's output: the root node's own; a +/// scalar root is exact when every operator it reads is, and has none +/// otherwise (#669). +fn root_guarantee(root: &QueryRoot) -> Option { + match root { + QueryRoot::Operator(node) => node.guarantee.clone(), + QueryRoot::Scalar(expr) => expr + .operator_refs() + .into_iter() + .all(|node| { + node.guarantee + .as_ref() + .is_some_and(ResultGuarantee::is_exact) + }) + .then(|| ResultGuarantee::exact("scalar over exact inputs")), + } +} + fn family_name(family: &FieldDataType) -> String { match family { FieldDataType::Sketch(kind, _) => format!("{:?}", kind.algorithm()), @@ -512,9 +549,10 @@ fn summary_shape(family: &FieldDataType) -> (u64, u64) { #[cfg(test)] mod tests { use super::*; + use crate::accuracy::{AccuracyModel, DefaultAccuracyModel, PropagationStats}; use crate::physical_candidates::stage2_physical; use crate::test_support::lower_promql; - use asap_types::ir::QueryRoot; + use asap_types::post_asap::{AccuracyError, CompositionOperator}; use asap_types::workload::{Evidence, Rate}; fn data() -> DataWorkload { @@ -548,7 +586,7 @@ mod tests { let topk = inventory .targets .iter() - .position(|t| t.candidates.len() > 2) + .position(|t| t.choices.candidates.len() > 2) .unwrap(); [0, 1, 2] .into_iter() @@ -560,10 +598,7 @@ mod tests { ) .unwrap() .into_iter() - .map(|(_, root)| match root { - QueryRoot::Operator(node) => node, - QueryRoot::Scalar(_) => panic!("operator root"), - }) + .map(|(_, root)| root) .collect(); let mut candidate = stage2_physical("L", &roots).unwrap(); candidate.id = format!("P{}", alternative + 1); @@ -656,4 +691,130 @@ mod tests { assert_eq!(costlier.len(), 1); assert_ne!(costlier[0].id, selection.selected); } + + /// Every combination of Pass 1 alternatives of one query, as Stage 2 + /// candidates `P1`, `P2`, ... + fn all_candidates(query: &str, target: &AccuracyTarget) -> Vec { + let root = lower_promql(query, target.clone()); + let inventory = crate::per_computation_candidates::generate_per_computation_candidates( + vec![(0, QueryRoot::Operator(root))], + ) + .unwrap(); + let mut choices = vec![vec![]]; + for target in &inventory.targets { + choices = choices + .into_iter() + .flat_map(|choice: Vec| { + (0..target.choices.candidates.len()).map(move |alternative| { + let mut choice = choice.clone(); + choice.push(alternative); + choice + }) + }) + .collect(); + } + choices + .iter() + .enumerate() + .map(|(index, choice)| { + let roots: Vec<_> = crate::per_computation_candidates::compose_logical_candidate( + &inventory, choice, + ) + .unwrap() + .into_iter() + .map(|(_, root)| root) + .collect(); + let mut candidate = stage2_physical("L", &roots).unwrap(); + candidate.id = format!("P{}", index + 1); + candidate + }) + .collect() + } + + fn has_summary_estimate(candidate: &PhysicalCandidate) -> bool { + candidate + .dag + .nodes + .iter() + .any(|n| matches!(n.payload, Payload::ASAP(ASAPOp::SummaryEstimate { .. }))) + } + + /// A deployment whose empirical accuracy shows every output exact. + struct EveryOutputMeasuredExact; + + impl AccuracyModel for EveryOutputMeasuredExact { + fn local_guarantee( + &self, + family: &FieldDataType, + query: &SketchStatistic, + ) -> Option { + DefaultAccuracyModel.local_guarantee(family, query) + } + fn propagate( + &self, + op: &CompositionOperator, + inputs: &[ResultGuarantee], + local: Option<&ResultGuarantee>, + stats: &PropagationStats, + ) -> Result { + DefaultAccuracyModel.propagate(op, inputs, local, stats) + } + fn satisfies(&self, guarantee: &ResultGuarantee, target: &AccuracyTarget) -> bool { + DefaultAccuracyModel.satisfies(guarantee, target) + } + fn output_guarantee(&self, _output: &QueryRoot) -> Option { + Some(ResultGuarantee::exact("measured")) + } + } + + /// A difference of two quantile estimates has no output guarantee, though + /// each estimate is certified: its summary candidates stay listed as + /// invalid with a reason. The deployment's empirical accuracy for the + /// output admits them. + #[test] + fn combined_estimates_need_evidence_about_the_output() { + let target = AccuracyTarget::EpsilonDelta { + epsilon: 0.01, + delta: 0.001, + }; + let candidates = all_candidates( + "quantile_over_time(0.99, m[1m]) - quantile_over_time(0.5, m[1m])", + &target, + ); + let summaries: Vec<_> = candidates + .iter() + .filter(|c| has_summary_estimate(c)) + .map(|c| c.id.as_str()) + .collect(); + assert!(!summaries.is_empty()); + + let builtin = stage3_select( + &candidates, + &[Some(target.clone())], + &data(), + PlanningModels::builtin(), + ) + .unwrap(); + for id in &summaries { + let rejection = builtin.rejected.iter().find(|r| r.id == *id).unwrap(); + assert!(!rejection.valid); + assert!( + rejection.reason.contains("no guarantee"), + "{}", + rejection.reason + ); + } + assert!(!summaries.contains(&builtin.selected.as_str())); + + let measured = stage3_select( + &candidates, + &[Some(target)], + &data(), + PlanningModels::builtin().with_accuracy(&EveryOutputMeasuredExact), + ) + .unwrap(); + for id in &summaries { + assert!(measured.costs.contains_key(*id), "{id} is valid"); + } + } } diff --git a/crates/devtools/src/bin/stage_pipeline.rs b/crates/devtools/src/bin/stage_pipeline.rs index 3ebc7f31..d406fccf 100644 --- a/crates/devtools/src/bin/stage_pipeline.rs +++ b/crates/devtools/src/bin/stage_pipeline.rs @@ -12,7 +12,7 @@ // (default 64). A candidate that cannot be composed carries `error` // instead of `dag`; // - stage2_physical_asap: one physical candidate per composed logical -// candidate (operator implementation only, everything at query time), no cost; +// candidate (operator implementation; every sub-DAG NotMaterialized), no cost; // - stage3_selection: per-candidate costs, the selected candidate, and // every other candidate as rejected (`valid: false`) or costlier. // @@ -130,17 +130,8 @@ fn stage_pipeline(workload: &PlanningWorkload, max_candidates: usize) -> Result< Ok(roots) => { let roots: Vec<_> = roots.into_iter().map(|(_, root)| root).collect(); candidate["dag"] = json!(export(&roots)?); - let operators = roots - .into_iter() - .map(|root| match root { - QueryRoot::Operator(node) => Ok(node), - QueryRoot::Scalar(_) => { - Err("Stage 2: scalar query roots are not physical yet") - } - }) - .collect::, _>>()?; let mut physical_candidate = - stage2_physical(&id, &operators).map_err(|e| format!("Stage 2 {id}: {e}"))?; + stage2_physical(&id, &roots).map_err(|e| format!("Stage 2 {id}: {e}"))?; physical_candidate.id = format!("P{}", index + 1); physical_candidate.label = label; physical.push(physical_candidate); From 0ab87ecbc2edddc2810a0036ee5b7c63bee23235 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 13 Oct 2026 03:10:02 +0000 Subject: [PATCH 05/17] test(devtools): regenerate the four-stage Example 1 fixture on the new #575 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W7qG9aFyPij5uWsyAJCxDW --- .../examples/planner-layering-example1.json | 5021 ++++++++++++++++- 1 file changed, 4962 insertions(+), 59 deletions(-) diff --git a/tools/dag-viewer/examples/planner-layering-example1.json b/tools/dag-viewer/examples/planner-layering-example1.json index 34a2b9f1..0a94b778 100644 --- a/tools/dag-viewer/examples/planner-layering-example1.json +++ b/tools/dag-viewer/examples/planner-layering-example1.json @@ -11,7 +11,7 @@ "Scan": { "predicates": [], "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -36,6 +36,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -51,7 +59,7 @@ }, "result_kind": "instant_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -76,6 +84,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -100,7 +116,7 @@ }, "result_kind": "range_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -125,6 +141,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -155,7 +179,7 @@ }, "result_kind": "instant_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -180,6 +204,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -251,7 +283,7 @@ "Scan": { "predicates": [], "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -276,6 +308,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -291,7 +331,7 @@ }, "result_kind": "instant_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -316,6 +356,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -340,7 +388,7 @@ }, "result_kind": "range_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -365,6 +413,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -396,7 +452,7 @@ }, "result_kind": "instant_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -421,6 +477,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -526,7 +590,7 @@ "Scan": { "predicates": [], "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -551,6 +615,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -566,7 +638,7 @@ }, "result_kind": "instant_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -591,6 +663,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -629,7 +709,7 @@ }, "result_kind": "range_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -654,6 +734,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -691,7 +779,7 @@ }, "result_kind": "state", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -719,6 +807,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -752,7 +848,7 @@ }, "result_kind": "instant_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -777,6 +873,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -925,7 +1029,7 @@ "Scan": { "predicates": [], "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -950,6 +1054,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -965,7 +1077,7 @@ }, "result_kind": "instant_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -990,6 +1102,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -1028,7 +1148,7 @@ }, "result_kind": "range_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -1053,6 +1173,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -1090,7 +1218,7 @@ }, "result_kind": "state", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -1118,6 +1246,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -1151,7 +1287,7 @@ }, "result_kind": "instant_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -1176,6 +1312,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -1295,7 +1439,7 @@ "Scan": { "predicates": [], "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -1320,6 +1464,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -1335,7 +1487,7 @@ }, "result_kind": "instant_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -1360,6 +1512,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -1398,7 +1558,7 @@ }, "result_kind": "range_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -1423,6 +1583,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -1460,7 +1628,7 @@ }, "result_kind": "state", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -1488,6 +1656,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -1521,7 +1697,7 @@ }, "result_kind": "instant_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -1546,6 +1722,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -1694,7 +1878,7 @@ "Scan": { "predicates": [], "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -1719,6 +1903,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -1734,7 +1926,7 @@ }, "result_kind": "instant_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -1759,6 +1951,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -1797,7 +1997,7 @@ }, "result_kind": "range_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -1822,6 +2022,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -1859,7 +2067,7 @@ }, "result_kind": "state", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -1887,6 +2095,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -1920,7 +2136,7 @@ }, "result_kind": "instant_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -1945,6 +2161,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -1979,14 +2203,8 @@ "grouping": "PerSubpopulationInstance", "input": { "item": { - "EntityIdentity": { - "PromqlLabelSet": { - "excluding": [ - { - "Named": "job" - } - ] - } + "Column": { + "Named": "$promql_series_identity" } }, "weight": { @@ -2185,7 +2403,7 @@ "Scan": { "predicates": [], "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -2210,6 +2428,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -2225,7 +2451,7 @@ }, "result_kind": "instant_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -2250,6 +2476,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -2288,7 +2522,7 @@ }, "result_kind": "range_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -2313,6 +2547,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -2350,7 +2592,7 @@ }, "result_kind": "state", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -2378,6 +2620,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -2411,7 +2661,7 @@ }, "result_kind": "instant_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -2436,11 +2686,19 @@ "name": "job", "nullable": true, "table": null - } - ], - "time_index": 0, - "unique_keys": [] - }, + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, "timing": null }, { @@ -2584,7 +2842,7 @@ "Scan": { "predicates": [], "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -2609,6 +2867,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -2624,7 +2890,7 @@ }, "result_kind": "instant_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -2649,6 +2915,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -2687,7 +2961,7 @@ }, "result_kind": "range_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -2712,6 +2986,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -2749,7 +3031,7 @@ }, "result_kind": "state", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -2777,6 +3059,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -2810,7 +3100,7 @@ }, "result_kind": "instant_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -2835,6 +3125,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -2869,14 +3167,8 @@ "grouping": "PerSubpopulationInstance", "input": { "item": { - "EntityIdentity": { - "PromqlLabelSet": { - "excluding": [ - { - "Named": "job" - } - ] - } + "Column": { + "Named": "$promql_series_identity" } }, "weight": { @@ -3054,6 +3346,4617 @@ "capped": false, "combinations": 3 }, + "stage2_physical_asap": { + "candidates": [ + { + "dag": { + "edges": [ + { + "consumer": 1, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 0, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 2, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 1, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 3, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 2, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 4, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 3, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 5, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "producer": 4, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 7, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 6, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 8, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 7, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 9, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 8, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 10, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 9, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 11, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 10, + "role": "Input", + "window": "NotApplicable" + } + ], + "nodes": [ + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 0, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Scan": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 1, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "TimeRange": { + "child": 0, + "kind": "range", + "range": { + "nanos": 0, + "secs": 60 + } + } + } + } + }, + { + "guarantee": null, + "id": 2, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 1, + "family": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": "PerEntity" + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "id": 3, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 2 + } + } + } + }, + { + "guarantee": null, + "id": 4, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 3, + "family": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": { + "Reduce": [ + 2 + ] + } + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "id": 5, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "sum", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 4 + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 6, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Scan": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 7, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "TimeRange": { + "child": 6, + "kind": "range", + "range": { + "nanos": 0, + "secs": 60 + } + } + } + } + }, + { + "guarantee": null, + "id": 8, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 7, + "family": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": "PerEntity" + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "id": 9, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 8 + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "id": 10, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Sort": { + "child": 9, + "keys": [ + { + "ascending": false, + "expr": { + "Column": 1 + }, + "nulls_first": false + } + ], + "partition_by": [ + 2 + ] + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 11, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Limit": { + "child": 10, + "n": 10, + "offset": 0, + "partition_by": [ + 2 + ] + } + } + } + } + ], + "roots": [ + 5, + 11 + ], + "scalar_roots": [] + }, + "from_logical": "L1", + "id": "P1", + "label": "Q1 exact (Sum acc, Rate acc) · Q2 exact (Sum acc)" + }, + { + "dag": { + "edges": [ + { + "consumer": 1, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 0, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 2, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 1, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 3, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 2, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 4, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 3, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 5, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "producer": 4, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 7, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 6, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 8, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 7, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 9, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 8, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 10, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 9, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 11, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Sketch": [ + { + "algorithm": "CmsWithHeap", + "category": "TopK", + "params": { + "CmsWithHeap": { + "depth": 7, + "heap_size": 100, + "width": 272 + } + } + }, + "PerSubpopulationInstance" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "producer": 10, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 12, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "topk", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "producer": 11, + "role": "Input", + "window": "NotApplicable" + } + ], + "nodes": [ + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 0, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Scan": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 1, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "TimeRange": { + "child": 0, + "kind": "range", + "range": { + "nanos": 0, + "secs": 60 + } + } + } + } + }, + { + "guarantee": null, + "id": 2, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 1, + "family": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": "PerEntity" + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "id": 3, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 2 + } + } + } + }, + { + "guarantee": null, + "id": 4, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 3, + "family": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": { + "Reduce": [ + 2 + ] + } + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "id": 5, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "sum", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 4 + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 6, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Scan": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 7, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "TimeRange": { + "child": 6, + "kind": "range", + "range": { + "nanos": 0, + "secs": 60 + } + } + } + } + }, + { + "guarantee": null, + "id": 8, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 7, + "family": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": "PerEntity" + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "id": 9, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 8 + } + } + } + }, + { + "guarantee": null, + "id": 10, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Sketch": [ + { + "algorithm": "CmsWithHeap", + "category": "TopK", + "params": { + "CmsWithHeap": { + "depth": 7, + "heap_size": 100, + "width": 272 + } + } + }, + "PerSubpopulationInstance" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 9, + "family": { + "Sketch": [ + { + "algorithm": "CmsWithHeap", + "category": "TopK", + "params": { + "CmsWithHeap": { + "depth": 7, + "heap_size": 100, + "width": 272 + } + } + }, + "PerSubpopulationInstance" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": { + "Column": { + "Named": "$promql_series_identity" + } + }, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": { + "Reduce": [ + 2 + ] + } + } + } + } + }, + { + "guarantee": null, + "id": 11, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "topk", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryEstimate": { + "query": { + "TopK": { + "k": 10 + } + }, + "summary_input": 10 + } + } + } + }, + { + "guarantee": null, + "id": 12, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "topk_10", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Project": { + "child": 11, + "cols": [ + { + "alias": "job", + "expr": { + "Column": 0 + } + }, + { + "alias": "topk_10", + "expr": { + "Column": 1 + } + } + ], + "qualifier": null + } + } + } + } + ], + "roots": [ + 5, + 12 + ], + "scalar_roots": [] + }, + "from_logical": "L2", + "id": "P2", + "label": "Q1 exact (Sum acc, Rate acc) · Q2 CMS+heap (Sum acc)" + }, + { + "dag": { + "edges": [ + { + "consumer": 1, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 0, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 2, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 1, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 3, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 2, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 4, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 3, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 5, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "producer": 4, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 7, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 6, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 8, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 7, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 9, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 8, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 10, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 9, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 11, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Sketch": [ + { + "algorithm": "CountSketchWithHeap", + "category": "TopK", + "params": { + "CountSketchWithHeap": { + "depth": 125, + "heap_size": 100, + "width": 30000 + } + } + }, + "PerSubpopulationInstance" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "producer": 10, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 12, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "topk", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "producer": 11, + "role": "Input", + "window": "NotApplicable" + } + ], + "nodes": [ + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 0, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Scan": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 1, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "TimeRange": { + "child": 0, + "kind": "range", + "range": { + "nanos": 0, + "secs": 60 + } + } + } + } + }, + { + "guarantee": null, + "id": 2, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 1, + "family": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": "PerEntity" + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "id": 3, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 2 + } + } + } + }, + { + "guarantee": null, + "id": 4, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 3, + "family": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": { + "Reduce": [ + 2 + ] + } + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "id": 5, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "sum", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 4 + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 6, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Scan": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 7, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "TimeRange": { + "child": 6, + "kind": "range", + "range": { + "nanos": 0, + "secs": 60 + } + } + } + } + }, + { + "guarantee": null, + "id": 8, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 7, + "family": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": "PerEntity" + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "id": 9, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 8 + } + } + } + }, + { + "guarantee": null, + "id": 10, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Sketch": [ + { + "algorithm": "CountSketchWithHeap", + "category": "TopK", + "params": { + "CountSketchWithHeap": { + "depth": 125, + "heap_size": 100, + "width": 30000 + } + } + }, + "PerSubpopulationInstance" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 9, + "family": { + "Sketch": [ + { + "algorithm": "CountSketchWithHeap", + "category": "TopK", + "params": { + "CountSketchWithHeap": { + "depth": 125, + "heap_size": 100, + "width": 30000 + } + } + }, + "PerSubpopulationInstance" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": { + "Column": { + "Named": "$promql_series_identity" + } + }, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": { + "Reduce": [ + 2 + ] + } + } + } + } + }, + { + "guarantee": null, + "id": 11, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "topk", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryEstimate": { + "query": { + "TopK": { + "k": 10 + } + }, + "summary_input": 10 + } + } + } + }, + { + "guarantee": null, + "id": 12, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "topk_10", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Project": { + "child": 11, + "cols": [ + { + "alias": "job", + "expr": { + "Column": 0 + } + }, + { + "alias": "topk_10", + "expr": { + "Column": 1 + } + } + ], + "qualifier": null + } + } + } + } + ], + "roots": [ + 5, + 12 + ], + "scalar_roots": [] + }, + "from_logical": "L3", + "id": "P3", + "label": "Q1 exact (Sum acc, Rate acc) · Q2 CountSketch+heap (Sum acc)" + } + ] + }, + "stage3_selection": { + "costs": { + "P1": { + "per_node": { + "0": { + "cost": 23.2, + "detail": "scan 4000000 samples" + }, + "1": { + "cost": 4.0, + "detail": "time range 60s: pass 4000000 rows" + }, + "10": { + "cost": 14.0, + "detail": "sort 1000000 rows in 100 partitions" + }, + "11": { + "cost": 0.001, + "detail": "limit to 1000 rows" + }, + "2": { + "cost": 4.0, + "detail": "build exact Rate accumulator into 1000000 states: 4000000 rows x depth 1" + }, + "3": { + "cost": 1.0, + "detail": "finalize 1000000 accumulators" + }, + "4": { + "cost": 1.0, + "detail": "build exact Sum accumulator into 100 states: 1000000 rows x depth 1" + }, + "5": { + "cost": 0.00009999999999999999, + "detail": "finalize 100 accumulators" + }, + "6": { + "cost": 23.2, + "detail": "scan 4000000 samples" + }, + "7": { + "cost": 4.0, + "detail": "time range 60s: pass 4000000 rows" + }, + "8": { + "cost": 4.0, + "detail": "build exact Sum accumulator into 1000000 states: 4000000 rows x depth 1" + }, + "9": { + "cost": 1.0, + "detail": "finalize 1000000 accumulators" + } + }, + "source": "analytical-cost-v1 (illustrative statistics)", + "total": 79.40110000000001, + "unit": "cpu_ms_per_workload_evaluation" + } + }, + "rejected": [ + { + "id": "P2", + "reason": "q2: CmsWithHeap needs non-negative update weights, and these are not proven non-negative", + "valid": false + }, + { + "id": "P3", + "reason": "q2: the output has no guarantee, and the accuracy model has no evidence that it meets target EpsilonDelta { epsilon: 0.01, delta: 0.001 }", + "valid": false + } + ], + "selected": "P1" + }, "workload": { "queries": [ { From b1792b3d61c0479e3db76b110ba805abda986919 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 13 Oct 2026 03:13:14 +0000 Subject: [PATCH 06/17] docs(planner): name Stage 2/3 by the planner-layering stage terms Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W7qG9aFyPij5uWsyAJCxDW --- crates/asap-aware-mapping/src/lib.rs | 7 +++++-- .../src/physical_candidates.rs | 18 +++++++++------- .../asap-aware-mapping/src/plan_selection.rs | 21 ++++++++++--------- 3 files changed, 27 insertions(+), 19 deletions(-) diff --git a/crates/asap-aware-mapping/src/lib.rs b/crates/asap-aware-mapping/src/lib.rs index 62b1400f..14db7b17 100644 --- a/crates/asap-aware-mapping/src/lib.rs +++ b/crates/asap-aware-mapping/src/lib.rs @@ -216,8 +216,11 @@ pub mod maintained_population; /// assigned: that is a Stage 2 materialization decision. pub mod per_computation_candidates; -/// #509 Stage 2 MVP: physical operator implementation, all at query time. +/// #509 Stage 2, physical ASAP-aware optimization (MVP): materialization +/// (every sub-DAG `NotMaterialized`) and physical operator implementation, +/// producing the CandidatePhysicalASAPDAGs. pub mod physical_candidates; -/// #509 Stage 3 MVP: accuracy check, analytical pricing, cheapest valid plan. +/// #509 Stage 3, plan selection (MVP): accuracy check, analytical pricing, +/// cheapest valid plan. pub mod plan_selection; diff --git a/crates/asap-aware-mapping/src/physical_candidates.rs b/crates/asap-aware-mapping/src/physical_candidates.rs index d35451f4..1db9d905 100644 --- a/crates/asap-aware-mapping/src/physical_candidates.rs +++ b/crates/asap-aware-mapping/src/physical_candidates.rs @@ -1,7 +1,9 @@ -//! #509 Stage 2 (MVP): physical operator implementation of one logical -//! candidate. Materialization offers one option for now: every sub-DAG is -//! `Materialization::NotMaterialized` and runs at query time, so each -//! logical candidate yields exactly one physical candidate. Enumerating +//! #509 Stage 2, physical ASAP-aware optimization (MVP): turns one +//! CandidateLogicalASAPDAG into CandidatePhysicalASAPDAGs by materialization +//! and physical operator implementation. Materialization offers one option +//! for now: every sub-DAG is `Materialization::NotMaterialized` and runs at +//! query time, so each logical candidate yields exactly one physical +//! candidate. Enumerating //! `MatAtIngestion(medium)` and `MatAtQuery(medium)` (Algorithm 3) comes //! later. //! @@ -27,7 +29,8 @@ use asap_types::pre_asap::expr_ir::ColumnRef; use asap_types::pre_asap::{AggIntent, Reduction}; use thiserror::Error; -/// One Stage 2 candidate, derived from exactly one Stage 1 candidate. +/// One CandidatePhysicalASAPDAG (Stage 2 output), derived from exactly one +/// CandidateLogicalASAPDAG (Stage 1 output). /// `roots` are the timed query roots (one per query, in workload order) /// that `dag` exports. #[derive(Debug, Clone)] @@ -49,7 +52,8 @@ pub enum Stage2Error { NoValueColumn(String), } -/// Implement every operator of one logical candidate and time it with every +/// Stage 2 for one CandidateLogicalASAPDAG: implement every operator +/// (physical operator implementation) and time it with every /// sub-DAG `NotMaterialized`. `id` and `label` are left empty for the caller /// to name. Sharing between `roots` is preserved: one memo serves the whole /// workload. @@ -267,7 +271,7 @@ mod tests { vec![(0, QueryRoot::Operator(root))], ) .unwrap(); - // A summary (the last alternative) for every target. + // A summary (the last candidate) for every target. let choice: Vec<_> = inventory .targets .iter() diff --git a/crates/asap-aware-mapping/src/plan_selection.rs b/crates/asap-aware-mapping/src/plan_selection.rs index 1b798240..022ba34e 100644 --- a/crates/asap-aware-mapping/src/plan_selection.rs +++ b/crates/asap-aware-mapping/src/plan_selection.rs @@ -1,7 +1,7 @@ -//! #509 Stage 3 (MVP): plan selection, the only stage that computes cost. +//! #509 Stage 3, plan selection (MVP): the only stage that computes cost. //! -//! Each Stage 2 candidate is checked against every query's accuracy target: -//! the query's output meets it when its root guarantee (composed in Stage 1) +//! Each CandidatePhysicalASAPDAG from Stage 2 is checked against every +//! query's accuracy target: the query's output meets it when its root guarantee (composed in Stage 1) //! does, or when the deployment's accuracy model shows this candidate's //! output does. Count-Min is admitted only over weights proven non-negative. //! A miss rejects the candidate as invalid, with a reason; only this stage @@ -100,7 +100,8 @@ pub enum SelectionError { NoValidCandidate(Vec), } -/// Reject candidates that miss a query's accuracy target, price the rest and +/// Stage 3 (plan selection): reject candidates that miss a query's accuracy +/// target, price the rest and /// select the cheapest (the first on ties). `targets[i]` is the requirement /// of `candidate.roots[i]`; `None` imposes none. pub fn stage3_select( @@ -590,9 +591,9 @@ mod tests { .unwrap(); [0, 1, 2] .into_iter() - .map(|alternative| { + .map(|index| { let mut choice = vec![0; inventory.targets.len()]; - choice[topk] = alternative; + choice[topk] = index; let roots: Vec<_> = crate::per_computation_candidates::compose_logical_candidate( &inventory, &choice, ) @@ -601,7 +602,7 @@ mod tests { .map(|(_, root)| root) .collect(); let mut candidate = stage2_physical("L", &roots).unwrap(); - candidate.id = format!("P{}", alternative + 1); + candidate.id = format!("P{}", index + 1); candidate }) .collect() @@ -692,7 +693,7 @@ mod tests { assert_ne!(costlier[0].id, selection.selected); } - /// Every combination of Pass 1 alternatives of one query, as Stage 2 + /// Every combination of Pass 1 candidates of one query, as Stage 2 /// candidates `P1`, `P2`, ... fn all_candidates(query: &str, target: &AccuracyTarget) -> Vec { let root = lower_promql(query, target.clone()); @@ -705,9 +706,9 @@ mod tests { choices = choices .into_iter() .flat_map(|choice: Vec| { - (0..target.choices.candidates.len()).map(move |alternative| { + (0..target.choices.candidates.len()).map(move |index| { let mut choice = choice.clone(); - choice.push(alternative); + choice.push(index); choice }) }) From 2a83ea7df7b16912d8c377b0f04568eb8f658e13 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 13 Oct 2026 03:24:32 +0000 Subject: [PATCH 07/17] fix(devtools): prune a logical candidate Stage 2 cannot implement, with its reason A Stage 2 error no longer fails the whole candidate set: stage_pipeline lists that candidate under stage2_physical_asap.pruned and plans the rest. Full materialization enumeration is left to #604. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W7qG9aFyPij5uWsyAJCxDW --- .../src/physical_candidates.rs | 6 +++-- crates/devtools/src/bin/stage_pipeline.rs | 22 +++++++++++++------ crates/devtools/tests/stage_pipeline.rs | 17 +++++++++----- .../examples/planner-layering-example1.json | 3 ++- 4 files changed, 33 insertions(+), 15 deletions(-) diff --git a/crates/asap-aware-mapping/src/physical_candidates.rs b/crates/asap-aware-mapping/src/physical_candidates.rs index 1db9d905..3171e204 100644 --- a/crates/asap-aware-mapping/src/physical_candidates.rs +++ b/crates/asap-aware-mapping/src/physical_candidates.rs @@ -4,8 +4,10 @@ //! for now: every sub-DAG is `Materialization::NotMaterialized` and runs at //! query time, so each logical candidate yields exactly one physical //! candidate. Enumerating -//! `MatAtIngestion(medium)` and `MatAtQuery(medium)` (Algorithm 3) comes -//! later. +//! `MatAtIngestion(medium)` and `MatAtQuery(medium)` per sub-DAG +//! (Algorithm 3) comes in #604. A candidate this stage cannot implement +//! fails with a [`Stage2Error`], and the caller prunes that candidate alone, +//! with the error as its reason. //! //! The runtime has one implementation per logical operator except exact //! top-k, which it cannot run as `Aggregate{[TopK]}`. Stage 2 records that diff --git a/crates/devtools/src/bin/stage_pipeline.rs b/crates/devtools/src/bin/stage_pipeline.rs index d406fccf..accc92fa 100644 --- a/crates/devtools/src/bin/stage_pipeline.rs +++ b/crates/devtools/src/bin/stage_pipeline.rs @@ -12,7 +12,9 @@ // (default 64). A candidate that cannot be composed carries `error` // instead of `dag`; // - stage2_physical_asap: one physical candidate per composed logical -// candidate (operator implementation; every sub-DAG NotMaterialized), no cost; +// candidate (operator implementation; every sub-DAG NotMaterialized), no +// cost; a composed logical candidate Stage 2 cannot implement is listed +// under `pruned` with the reason; // - stage3_selection: per-candidate costs, the selected candidate, and // every other candidate as rejected (`valid: false`) or costlier. // @@ -117,6 +119,7 @@ fn stage_pipeline(workload: &PlanningWorkload, max_candidates: usize) -> Result< let owners = target_owners(&inventory); let mut candidates = Vec::new(); let mut physical = Vec::new(); + let mut pruned = Vec::new(); let mut choice = vec![0; inventory.targets.len()]; for index in 0..combinations.min(max_candidates) { let (id, label) = ( @@ -130,11 +133,16 @@ fn stage_pipeline(workload: &PlanningWorkload, max_candidates: usize) -> Result< Ok(roots) => { let roots: Vec<_> = roots.into_iter().map(|(_, root)| root).collect(); candidate["dag"] = json!(export(&roots)?); - let mut physical_candidate = - stage2_physical(&id, &roots).map_err(|e| format!("Stage 2 {id}: {e}"))?; - physical_candidate.id = format!("P{}", index + 1); - physical_candidate.label = label; - physical.push(physical_candidate); + match stage2_physical(&id, &roots) { + Ok(mut physical_candidate) => { + physical_candidate.id = format!("P{}", index + 1); + physical_candidate.label = label; + physical.push(physical_candidate); + } + Err(e) => pruned.push( + json!({ "from_logical": id, "label": label, "reason": e.to_string() }), + ), + } } Err(error) => candidate["error"] = json!(error.to_string()), } @@ -169,7 +177,7 @@ fn stage_pipeline(workload: &PlanningWorkload, max_candidates: usize) -> Result< "capped": combinations > max_candidates, "candidates": candidates, }, - "stage2_physical_asap": { "candidates": stage2 }, + "stage2_physical_asap": { "candidates": stage2, "pruned": pruned }, "stage3_selection": stage3_json(&selection), })) } diff --git a/crates/devtools/tests/stage_pipeline.rs b/crates/devtools/tests/stage_pipeline.rs index fbd56e23..9b8f6caa 100644 --- a/crates/devtools/tests/stage_pipeline.rs +++ b/crates/devtools/tests/stage_pipeline.rs @@ -47,10 +47,10 @@ fn validated_dag(value: &Value, queries: usize) { } /// Every logical DAG is a well-formed flat DAG and every physical DAG validates; -/// each has one root per query; ids are unique; -/// Stage 2 maps the composed logical candidates one-to-one with no cost; -/// Stage 3 accounts for every Stage 2 candidate once and costs exactly the -/// valid ones; the committed fixture is current. +/// each has one root per query; ids are unique; Stage 2 implements or prunes +/// (with a reason) each composed logical candidate, with no cost; Stage 3 +/// accounts for every Stage 2 candidate once and costs exactly the valid ones; +/// the committed fixture is current. #[test] fn example1_document_is_valid_and_committed_fixture_is_current() { let document = generate(&[]); @@ -84,11 +84,18 @@ fn example1_document_is_valid_and_committed_fixture_is_current() { let physical = document["stage2_physical_asap"]["candidates"] .as_array() .unwrap(); + let pruned = document["stage2_physical_asap"]["pruned"] + .as_array() + .unwrap(); + assert!(pruned + .iter() + .all(|p| !p["reason"].as_str().unwrap().is_empty())); let sources: HashSet<_> = physical .iter() + .chain(pruned) .map(|p| p["from_logical"].as_str().unwrap()) .collect(); - assert_eq!(physical.len(), composed.len()); + assert_eq!(physical.len() + pruned.len(), composed.len()); assert_eq!(sources, composed); for candidate in physical { assert!(candidate.get("cost").is_none(), "Stage 2 has no cost"); diff --git a/tools/dag-viewer/examples/planner-layering-example1.json b/tools/dag-viewer/examples/planner-layering-example1.json index 0a94b778..d9cb5928 100644 --- a/tools/dag-viewer/examples/planner-layering-example1.json +++ b/tools/dag-viewer/examples/planner-layering-example1.json @@ -7883,7 +7883,8 @@ "id": "P3", "label": "Q1 exact (Sum acc, Rate acc) · Q2 CountSketch+heap (Sum acc)" } - ] + ], + "pruned": [] }, "stage3_selection": { "costs": { From 7fcd2af8ed10c4ad8e8ad241568a19aa12389c6e Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 13 Oct 2026 03:37:00 +0000 Subject: [PATCH 08/17] fix(planner): read empirical output accuracy from deployment evidence in Stage 3 The output-level hook moves from AccuracyModel to AccuracyEvidenceProvider: AccuracyModel stays theoretical, Stage 1 composes root guarantees with the built-in model, and deployment evidence enters only in Stage 3. A scalar root that is exactly scalar(v) inherits v's guarantee; any other scalar expression over an approximate operand has none until #669. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W7qG9aFyPij5uWsyAJCxDW --- .../src/accuracy/evidence.rs | 12 +++ crates/asap-aware-mapping/src/accuracy/mod.rs | 12 +-- .../asap-aware-mapping/src/plan_selection.rs | 95 +++++++++++-------- 3 files changed, 71 insertions(+), 48 deletions(-) diff --git a/crates/asap-aware-mapping/src/accuracy/evidence.rs b/crates/asap-aware-mapping/src/accuracy/evidence.rs index f779d578..f770dd80 100644 --- a/crates/asap-aware-mapping/src/accuracy/evidence.rs +++ b/crates/asap-aware-mapping/src/accuracy/evidence.rs @@ -1,5 +1,6 @@ //! Scoped source contracts and evidence required by accuracy rules. use super::*; +use asap_types::ir::QueryRoot; /// A trusted source assertion scoped by `AccuracyEvidenceProvider` to one /// complete evaluation. Choosing this variant asserts the estimator and hash @@ -106,6 +107,17 @@ pub trait AccuracyEvidenceProvider { None } + /// The deployment's empirical accuracy for one candidate's whole query + /// output, as a guarantee. Plan selection (#509 Stage 3) accepts a + /// candidate whose output has no theoretical guarantee that meets the + /// target (e.g. a difference of two quantile estimates) only when this + /// shows the output meets it. Stage 1 composition never reads it: root + /// guarantees there come from the built-in theoretical model. `None` + /// means no evidence. + fn output_guarantee(&self, _output: &QueryRoot) -> Option { + None + } + fn propagation_stats( &self, _op: &CompositionOperator, diff --git a/crates/asap-aware-mapping/src/accuracy/mod.rs b/crates/asap-aware-mapping/src/accuracy/mod.rs index 1c6d650c..a2f3354c 100644 --- a/crates/asap-aware-mapping/src/accuracy/mod.rs +++ b/crates/asap-aware-mapping/src/accuracy/mod.rs @@ -20,7 +20,7 @@ pub use evidence::{ QuantileInputDomain, WorkloadAccuracyEvidence, }; -use asap_types::ir::{OperatorNode, QueryRoot}; +use asap_types::ir::OperatorNode; use asap_types::post_asap::{ AccuracyError, BoundExpr, CompositionOperator, ErrorMetric, FieldDataType, GuaranteeSource, ProbabilityExpr, ResultGuarantee, SketchAlgorithm, SketchParams, SketchStatistic, @@ -65,16 +65,6 @@ pub trait AccuracyModel { stats: &PropagationStats, ) -> Result; - /// The deployment's empirical accuracy for one candidate's whole query - /// output, as a guarantee. Plan selection (#509 Stage 3) accepts a - /// candidate whose output has no theoretical guarantee that meets the - /// target, e.g. a difference of two quantile estimates, only when this - /// shows the output meets it. The built-in model has no empirical - /// evidence. - fn output_guarantee(&self, _output: &QueryRoot) -> Option { - None - } - /// Compare the dimensions requested by `target`. Unknown required /// dimensions fail; selection separately excludes missing accuracy evidence. fn satisfies(&self, guarantee: &ResultGuarantee, target: &AccuracyTarget) -> bool; diff --git a/crates/asap-aware-mapping/src/plan_selection.rs b/crates/asap-aware-mapping/src/plan_selection.rs index 022ba34e..af3a7103 100644 --- a/crates/asap-aware-mapping/src/plan_selection.rs +++ b/crates/asap-aware-mapping/src/plan_selection.rs @@ -1,9 +1,11 @@ //! #509 Stage 3, plan selection (MVP): the only stage that computes cost. //! //! Each CandidatePhysicalASAPDAG from Stage 2 is checked against every -//! query's accuracy target: the query's output meets it when its root guarantee (composed in Stage 1) -//! does, or when the deployment's accuracy model shows this candidate's -//! output does. Count-Min is admitted only over weights proven non-negative. +//! query's accuracy target: the query's output meets it when its root +//! guarantee does (theoretical: Stage 1 composes it with the built-in +//! accuracy model), or when the deployment's empirical evidence shows this +//! candidate's output does (`AccuracyEvidenceProvider::output_guarantee`, +//! read only here). Count-Min is admitted only over weights proven non-negative. //! A miss rejects the candidate as invalid, with a reason; only this stage //! discards valid candidates. //! Every valid candidate is priced node by node over its physical DAG, so a @@ -21,7 +23,7 @@ use std::rc::Rc; use asap_types::ir::physical_export::{ PhysicalASAPDAG, PhysicalASAPNodeId, PhysicalASAPOperatorPayload as Payload, }; -use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, QueryRoot}; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, QueryRoot, ScalarExpr}; use asap_types::post_asap::{ FieldDataType, ResultGuarantee, SketchAlgorithm, SketchParams, SketchStatistic, WeightDomain, }; @@ -162,9 +164,10 @@ pub fn stage3_select( } /// The first query whose output is not shown to meet its target, as a -/// reason. An output meets its target when its root guarantee (composed in -/// Stage 1) satisfies it, or when the deployment's accuracy model shows that -/// this candidate's output does ([`crate::accuracy::AccuracyModel::output_guarantee`]). A +/// reason. An output meets its target when its theoretical root guarantee +/// (composed in Stage 1 with the built-in model) satisfies it, or when the +/// deployment's empirical evidence shows that this candidate's output does +/// ([`crate::accuracy::AccuracyEvidenceProvider::output_guarantee`]). A /// certified estimate inside the plan is not enough on its own: a difference /// or ratio of certified estimates can be unbounded. Count-Min is admitted /// only over update weights proven non-negative, which its bound assumes. @@ -198,7 +201,7 @@ fn accuracy_violation( let meets = |g: &ResultGuarantee| models.accuracy.satisfies(g, target); if guarantee.as_ref().is_some_and(meets) || models - .accuracy + .evidence .output_guarantee(root) .as_ref() .is_some_and(meets) @@ -208,12 +211,12 @@ fn accuracy_violation( return Some(match guarantee { Some(g) => format!( "q{query}: the output guarantees bound {:?}, failure probability {:?}, which \ - misses target {target:?}, and the accuracy model has no evidence that it meets it", + misses target {target:?}, and the deployment has no evidence that it meets it", g.bound.evaluate(), g.failure_probability.evaluate(), ), None => format!( - "q{query}: the output has no guarantee, and the accuracy model has no evidence \ + "q{query}: the output has no guarantee, and the deployment has no evidence \ that it meets target {target:?}" ), }); @@ -229,12 +232,13 @@ fn root_operators(root: &QueryRoot) -> Vec<&Rc> { } } -/// The theoretical guarantee of a query's output: the root node's own; a -/// scalar root is exact when every operator it reads is, and has none -/// otherwise (#669). +/// The theoretical guarantee of a query's output: the root node's own; +/// `scalar(v)` alone inherits `v`'s; any other scalar root is exact when +/// every operator it reads is, and has none otherwise (#669). fn root_guarantee(root: &QueryRoot) -> Option { match root { QueryRoot::Operator(node) => node.guarantee.clone(), + QueryRoot::Scalar(ScalarExpr::PromqlScalarFromVector(node)) => node.guarantee.clone(), QueryRoot::Scalar(expr) => expr .operator_refs() .into_iter() @@ -550,10 +554,12 @@ fn summary_shape(family: &FieldDataType) -> (u64, u64) { #[cfg(test)] mod tests { use super::*; - use crate::accuracy::{AccuracyModel, DefaultAccuracyModel, PropagationStats}; + use crate::accuracy::AccuracyEvidenceProvider; use crate::physical_candidates::stage2_physical; use crate::test_support::lower_promql; - use asap_types::post_asap::{AccuracyError, CompositionOperator}; + use asap_types::ir::ExprSemantics; + use asap_types::post_asap::{BoundExpr, ErrorMetric, ProbabilityExpr}; + use asap_types::pre_asap::expr_ir::ArithmeticOpKind; use asap_types::workload::{Evidence, Rate}; fn data() -> DataWorkload { @@ -740,29 +746,10 @@ mod tests { .any(|n| matches!(n.payload, Payload::ASAP(ASAPOp::SummaryEstimate { .. }))) } - /// A deployment whose empirical accuracy shows every output exact. + /// Deployment evidence that every output was measured exact. struct EveryOutputMeasuredExact; - impl AccuracyModel for EveryOutputMeasuredExact { - fn local_guarantee( - &self, - family: &FieldDataType, - query: &SketchStatistic, - ) -> Option { - DefaultAccuracyModel.local_guarantee(family, query) - } - fn propagate( - &self, - op: &CompositionOperator, - inputs: &[ResultGuarantee], - local: Option<&ResultGuarantee>, - stats: &PropagationStats, - ) -> Result { - DefaultAccuracyModel.propagate(op, inputs, local, stats) - } - fn satisfies(&self, guarantee: &ResultGuarantee, target: &AccuracyTarget) -> bool { - DefaultAccuracyModel.satisfies(guarantee, target) - } + impl AccuracyEvidenceProvider for EveryOutputMeasuredExact { fn output_guarantee(&self, _output: &QueryRoot) -> Option { Some(ResultGuarantee::exact("measured")) } @@ -811,11 +798,45 @@ mod tests { &candidates, &[Some(target)], &data(), - PlanningModels::builtin().with_accuracy(&EveryOutputMeasuredExact), + PlanningModels::builtin().with_evidence(&EveryOutputMeasuredExact), ) .unwrap(); for id in &summaries { assert!(measured.costs.contains_key(*id), "{id} is valid"); } } + + /// `scalar(v)` alone inherits `v`'s guarantee; any other scalar + /// expression over an approximate operand has none until #669; one over + /// exact operands only is exact. + #[test] + fn scalar_of_one_vector_inherits_its_guarantee_and_nothing_else_does() { + let node = lower_promql("sum(m)", AccuracyTarget::Exact); + let approximate = ResultGuarantee { + metric: ErrorMetric::AbsoluteValue, + bound: BoundExpr::Constant { value: 0.01 }, + failure_probability: ProbabilityExpr::Constant { value: 0.001 }, + provenance: vec![], + }; + let with = |guarantee: Option| { + ScalarExpr::PromqlScalarFromVector(Rc::new((*node).clone().with_guarantee(guarantee))) + }; + let doubled = |operand: ScalarExpr| { + QueryRoot::Scalar(ScalarExpr::Arithmetic { + op: ArithmeticOpKind::Mul, + left: Box::new(operand), + right: Box::new(ScalarExpr::literal_f64(2.0)), + semantics: ExprSemantics::Promql, + }) + }; + assert_eq!( + root_guarantee(&QueryRoot::Scalar(with(Some(approximate.clone())))), + Some(approximate.clone()) + ); + assert_eq!(root_guarantee(&doubled(with(Some(approximate)))), None); + assert!( + root_guarantee(&doubled(with(Some(ResultGuarantee::exact("exact"))))) + .is_some_and(|g| g.is_exact()) + ); + } } From 0608e33f22257c6994a479d7970d0c32705c8f9a Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 13 Oct 2026 03:56:04 +0000 Subject: [PATCH 09/17] refactor(planner): name Stage 2/3 code and pipeline keys after the stage names physical_candidates -> physical_asap_optimization (stage2_physical -> optimize_physical_asap, Stage2Error -> PhysicalASAPOptimizationError), stage3_select -> select_plan, and the stage_pipeline keys stage2_physical_asap -> physical_asap_optimization and stage3_selection -> plan_selection. The fixture is regenerated. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W7qG9aFyPij5uWsyAJCxDW --- crates/asap-aware-mapping/src/lib.rs | 2 +- ...dates.rs => physical_asap_optimization.rs} | 28 +++++++++++-------- .../asap-aware-mapping/src/plan_selection.rs | 20 ++++++------- crates/devtools/src/bin/stage_pipeline.rs | 20 ++++++------- crates/devtools/tests/stage_pipeline.rs | 12 ++++---- .../examples/planner-layering-example1.json | 6 ++-- 6 files changed, 47 insertions(+), 41 deletions(-) rename crates/asap-aware-mapping/src/{physical_candidates.rs => physical_asap_optimization.rs} (92%) diff --git a/crates/asap-aware-mapping/src/lib.rs b/crates/asap-aware-mapping/src/lib.rs index 14db7b17..44a0f645 100644 --- a/crates/asap-aware-mapping/src/lib.rs +++ b/crates/asap-aware-mapping/src/lib.rs @@ -219,7 +219,7 @@ pub mod per_computation_candidates; /// #509 Stage 2, physical ASAP-aware optimization (MVP): materialization /// (every sub-DAG `NotMaterialized`) and physical operator implementation, /// producing the CandidatePhysicalASAPDAGs. -pub mod physical_candidates; +pub mod physical_asap_optimization; /// #509 Stage 3, plan selection (MVP): accuracy check, analytical pricing, /// cheapest valid plan. diff --git a/crates/asap-aware-mapping/src/physical_candidates.rs b/crates/asap-aware-mapping/src/physical_asap_optimization.rs similarity index 92% rename from crates/asap-aware-mapping/src/physical_candidates.rs rename to crates/asap-aware-mapping/src/physical_asap_optimization.rs index 3171e204..d009fae6 100644 --- a/crates/asap-aware-mapping/src/physical_candidates.rs +++ b/crates/asap-aware-mapping/src/physical_asap_optimization.rs @@ -6,7 +6,7 @@ //! candidate. Enumerating //! `MatAtIngestion(medium)` and `MatAtQuery(medium)` per sub-DAG //! (Algorithm 3) comes in #604. A candidate this stage cannot implement -//! fails with a [`Stage2Error`], and the caller prunes that candidate alone, +//! fails with a [`PhysicalASAPOptimizationError`], and the caller prunes that candidate alone, //! with the error as its reason. //! //! The runtime has one implementation per logical operator except exact @@ -45,7 +45,7 @@ pub struct PhysicalCandidate { } #[derive(Debug, Error)] -pub enum Stage2Error { +pub enum PhysicalASAPOptimizationError { #[error(transparent)] Structure(#[from] SchemaDerivationError), #[error(transparent)] @@ -59,10 +59,10 @@ pub enum Stage2Error { /// sub-DAG `NotMaterialized`. `id` and `label` are left empty for the caller /// to name. Sharing between `roots` is preserved: one memo serves the whole /// workload. -pub fn stage2_physical( +pub fn optimize_physical_asap( from_logical: &str, roots: &[QueryRoot], -) -> Result { +) -> Result { let mut memo = HashMap::new(); let implemented = map_roots(roots, &mut |node| implement(node, &mut memo))?; let assignment = MaterializationAssignment::with_default(Materialization::NotMaterialized); @@ -87,8 +87,8 @@ pub fn stage2_physical( /// `roots` with every operator node they read replaced by `f`'s result. fn map_roots( roots: &[QueryRoot], - f: &mut impl FnMut(&Rc) -> Result, Stage2Error>, -) -> Result, Stage2Error> { + f: &mut impl FnMut(&Rc) -> Result, PhysicalASAPOptimizationError>, +) -> Result, PhysicalASAPOptimizationError> { roots .iter() .map(|root| match root { @@ -108,7 +108,10 @@ fn map_roots( type Memo = HashMap<*const OperatorNode, Rc>; -fn implement(node: &Rc, memo: &mut Memo) -> Result, Stage2Error> { +fn implement( + node: &Rc, + memo: &mut Memo, +) -> Result, PhysicalASAPOptimizationError> { if let Some(done) = memo.get(&Rc::as_ptr(node)) { return Ok(done.clone()); } @@ -127,7 +130,7 @@ fn implement(node: &Rc, memo: &mut Memo) -> Result { let child = memo[&Rc::as_ptr(child)].clone(); let value = resolve_column_ref(&ColumnRef::SampleValue, &child.schema) - .map_err(|e| Stage2Error::NoValueColumn(e.to_string()))?; + .map_err(|e| PhysicalASAPOptimizationError::NoValueColumn(e.to_string()))?; let guarantee = child.guarantee.clone(); let sorted = Rc::new( OperatorNode::new(Operator::NonASAP(NonASAPOp::Sort { @@ -160,7 +163,10 @@ fn implement(node: &Rc, memo: &mut Memo) -> Result, memo: &Memo) -> Result, Stage2Error> { +fn rebuilt( + node: &Rc, + memo: &Memo, +) -> Result, PhysicalASAPOptimizationError> { let changed = node .children() .iter() @@ -197,7 +203,7 @@ mod tests { panic!("topk aggregate") }; let sum = Rc::new(sum.with_new_children(|_| shared.clone()).unwrap()); - let candidate = stage2_physical("L1", &[topk.into(), sum.into()]).unwrap(); + let candidate = optimize_physical_asap("L1", &[topk.into(), sum.into()]).unwrap(); let root = |i: usize| match &candidate.roots[i] { QueryRoot::Operator(node) => node.clone(), QueryRoot::Scalar(_) => panic!("operator root"), @@ -285,7 +291,7 @@ mod tests { .into_iter() .map(|(_, root)| root) .collect(); - let candidate = stage2_physical("L1", &roots).unwrap(); + let candidate = optimize_physical_asap("L1", &roots).unwrap(); assert!(candidate .dag .nodes diff --git a/crates/asap-aware-mapping/src/plan_selection.rs b/crates/asap-aware-mapping/src/plan_selection.rs index af3a7103..ba5050c4 100644 --- a/crates/asap-aware-mapping/src/plan_selection.rs +++ b/crates/asap-aware-mapping/src/plan_selection.rs @@ -35,7 +35,7 @@ use thiserror::Error; use crate::analytical_cost::{ estimate_operator, AnalyticalCostError, PhysicalOperator, ResourceCalibration, ResourceEstimate, }; -use crate::physical_candidates::PhysicalCandidate; +use crate::physical_asap_optimization::PhysicalCandidate; use crate::physical_operator_statistics::{ EdgeStatistics, OperatorStatistics, PartitionStatistics, UnaryEdgeStatistics, }; @@ -106,7 +106,7 @@ pub enum SelectionError { /// target, price the rest and /// select the cheapest (the first on ties). `targets[i]` is the requirement /// of `candidate.roots[i]`; `None` imposes none. -pub fn stage3_select( +pub fn select_plan( cands: &[PhysicalCandidate], targets: &[Option], data: &DataWorkload, @@ -555,7 +555,7 @@ fn summary_shape(family: &FieldDataType) -> (u64, u64) { mod tests { use super::*; use crate::accuracy::AccuracyEvidenceProvider; - use crate::physical_candidates::stage2_physical; + use crate::physical_asap_optimization::optimize_physical_asap; use crate::test_support::lower_promql; use asap_types::ir::ExprSemantics; use asap_types::post_asap::{BoundExpr, ErrorMetric, ProbabilityExpr}; @@ -607,7 +607,7 @@ mod tests { .into_iter() .map(|(_, root)| root) .collect(); - let mut candidate = stage2_physical("L", &roots).unwrap(); + let mut candidate = optimize_physical_asap("L", &roots).unwrap(); candidate.id = format!("P{}", index + 1); candidate }) @@ -623,7 +623,7 @@ mod tests { epsilon: 1e-6, delta: 1e-9, }; - let selection = stage3_select( + let selection = select_plan( &candidates, &[Some(strict)], &data(), @@ -654,7 +654,7 @@ mod tests { epsilon: 0.01, delta: 0.001, }; - let selection = stage3_select( + let selection = select_plan( &candidates(), &[Some(target)], &data(), @@ -675,7 +675,7 @@ mod tests { epsilon: 0.01, delta: 0.001, }; - let selection = stage3_select( + let selection = select_plan( &candidates, &[Some(target)], &data(), @@ -731,7 +731,7 @@ mod tests { .into_iter() .map(|(_, root)| root) .collect(); - let mut candidate = stage2_physical("L", &roots).unwrap(); + let mut candidate = optimize_physical_asap("L", &roots).unwrap(); candidate.id = format!("P{}", index + 1); candidate }) @@ -776,7 +776,7 @@ mod tests { .collect(); assert!(!summaries.is_empty()); - let builtin = stage3_select( + let builtin = select_plan( &candidates, &[Some(target.clone())], &data(), @@ -794,7 +794,7 @@ mod tests { } assert!(!summaries.contains(&builtin.selected.as_str())); - let measured = stage3_select( + let measured = select_plan( &candidates, &[Some(target)], &data(), diff --git a/crates/devtools/src/bin/stage_pipeline.rs b/crates/devtools/src/bin/stage_pipeline.rs index accc92fa..81175386 100644 --- a/crates/devtools/src/bin/stage_pipeline.rs +++ b/crates/devtools/src/bin/stage_pipeline.rs @@ -11,11 +11,11 @@ // product), in enumeration order and capped by `--max-candidates` // (default 64). A candidate that cannot be composed carries `error` // instead of `dag`; -// - stage2_physical_asap: one physical candidate per composed logical +// - physical_asap_optimization: one physical candidate per composed logical // candidate (operator implementation; every sub-DAG NotMaterialized), no // cost; a composed logical candidate Stage 2 cannot implement is listed // under `pruned` with the reason; -// - stage3_selection: per-candidate costs, the selected candidate, and +// - plan_selection: per-candidate costs, the selected candidate, and // every other candidate as rejected (`valid: false`) or costlier. // // `--promql` may repeat. `--epsilon`/`--delta` apply to every `--promql` @@ -28,8 +28,8 @@ use asap_aware_mapping::per_computation_candidates::{ compose_logical_candidate, generate_per_computation_candidates, ComputationCandidate, PerComputationCandidates, }; -use asap_aware_mapping::physical_candidates::stage2_physical; -use asap_aware_mapping::plan_selection::{stage3_select, Selection}; +use asap_aware_mapping::physical_asap_optimization::optimize_physical_asap; +use asap_aware_mapping::plan_selection::{select_plan, Selection}; use asap_aware_mapping::PlanningModels; use asap_types::ir::flat::{flatten, FlatDag}; use asap_types::ir::schema_support::with_promql_series_identity; @@ -133,7 +133,7 @@ fn stage_pipeline(workload: &PlanningWorkload, max_candidates: usize) -> Result< Ok(roots) => { let roots: Vec<_> = roots.into_iter().map(|(_, root)| root).collect(); candidate["dag"] = json!(export(&roots)?); - match stage2_physical(&id, &roots) { + match optimize_physical_asap(&id, &roots) { Ok(mut physical_candidate) => { physical_candidate.id = format!("P{}", index + 1); physical_candidate.label = label; @@ -162,9 +162,9 @@ fn stage_pipeline(workload: &PlanningWorkload, max_candidates: usize) -> Result< .map(|entry| Some(entry.requirements.accuracy.target())) .collect(); let data = workload.data_workload.clone().unwrap_or_default(); - let selection = stage3_select(&physical, &targets, &data, PlanningModels::builtin()) + let selection = select_plan(&physical, &targets, &data, PlanningModels::builtin()) .map_err(|e| format!("Stage 3: {e}"))?; - let stage2: Vec<_> = physical + let physical_json: Vec<_> = physical .iter() .map(|p| json!({ "id": p.id, "from_logical": p.from_logical, "label": p.label, "dag": p.dag })) .collect(); @@ -177,12 +177,12 @@ fn stage_pipeline(workload: &PlanningWorkload, max_candidates: usize) -> Result< "capped": combinations > max_candidates, "candidates": candidates, }, - "stage2_physical_asap": { "candidates": stage2, "pruned": pruned }, - "stage3_selection": stage3_json(&selection), + "physical_asap_optimization": { "candidates": physical_json, "pruned": pruned }, + "plan_selection": plan_selection_json(&selection), })) } -fn stage3_json(selection: &Selection) -> Value { +fn plan_selection_json(selection: &Selection) -> Value { let costs: serde_json::Map<_, _> = selection .costs .iter() diff --git a/crates/devtools/tests/stage_pipeline.rs b/crates/devtools/tests/stage_pipeline.rs index 9b8f6caa..897ab193 100644 --- a/crates/devtools/tests/stage_pipeline.rs +++ b/crates/devtools/tests/stage_pipeline.rs @@ -81,10 +81,10 @@ fn example1_document_is_valid_and_committed_fixture_is_current() { } } - let physical = document["stage2_physical_asap"]["candidates"] + let physical = document["physical_asap_optimization"]["candidates"] .as_array() .unwrap(); - let pruned = document["stage2_physical_asap"]["pruned"] + let pruned = document["physical_asap_optimization"]["pruned"] .as_array() .unwrap(); assert!(pruned @@ -104,11 +104,11 @@ fn example1_document_is_valid_and_committed_fixture_is_current() { PhysicalASAPDAGDocument::new(dag).validate().unwrap(); } - let stage3 = &document["stage3_selection"]; - let costs = stage3["costs"].as_object().unwrap(); - let selected = stage3["selected"].as_str().unwrap(); + let selection = &document["plan_selection"]; + let costs = selection["costs"].as_object().unwrap(); + let selected = selection["selected"].as_str().unwrap(); let mut accounted = HashSet::from([selected]); - for rejection in stage3["rejected"].as_array().unwrap() { + for rejection in selection["rejected"].as_array().unwrap() { let id = rejection["id"].as_str().unwrap(); assert!(accounted.insert(id), "{id} accounted for once"); assert!(!rejection["reason"].as_str().unwrap().is_empty()); diff --git a/tools/dag-viewer/examples/planner-layering-example1.json b/tools/dag-viewer/examples/planner-layering-example1.json index d9cb5928..24bc139c 100644 --- a/tools/dag-viewer/examples/planner-layering-example1.json +++ b/tools/dag-viewer/examples/planner-layering-example1.json @@ -3346,7 +3346,7 @@ "capped": false, "combinations": 3 }, - "stage2_physical_asap": { + "physical_asap_optimization": { "candidates": [ { "dag": { @@ -7886,7 +7886,7 @@ ], "pruned": [] }, - "stage3_selection": { + "plan_selection": { "costs": { "P1": { "per_node": { @@ -7952,7 +7952,7 @@ }, { "id": "P3", - "reason": "q2: the output has no guarantee, and the accuracy model has no evidence that it meets target EpsilonDelta { epsilon: 0.01, delta: 0.001 }", + "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target EpsilonDelta { epsilon: 0.01, delta: 0.001 }", "valid": false } ], From fa26db9df72f5517ca5412ddaed58b83210ab735 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 13 Oct 2026 04:19:59 +0000 Subject: [PATCH 10/17] test(planner): plan selection tests after Top-K estimates lost their bound #575 gives Top-K and signed Count-Min estimates guarantee None, so the CountSketch+heap candidate of the top-k fixture no longer carries an analytical guarantee: plan selection rejects it for missing evidence rather than for missing the target. The "misses target" case now uses a quantile summary planned at a looser target, and the per-node cost test admits the CountSketch+heap candidate through deployment evidence. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W7qG9aFyPij5uWsyAJCxDW --- .../asap-aware-mapping/src/plan_selection.rs | 46 +++++++++++-------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/crates/asap-aware-mapping/src/plan_selection.rs b/crates/asap-aware-mapping/src/plan_selection.rs index ba5050c4..8ffeab97 100644 --- a/crates/asap-aware-mapping/src/plan_selection.rs +++ b/crates/asap-aware-mapping/src/plan_selection.rs @@ -615,14 +615,25 @@ mod tests { } /// A summary whose analytical guarantee misses the target is rejected as - /// invalid with a reason; the exact plan is then selected. + /// invalid with a reason; the exact plan is then selected. (Top-K + /// estimates carry no guarantee since #575, so a quantile summary is used.) #[test] fn accuracy_failing_candidate_is_rejected_as_invalid() { - let candidates = candidates(); + let planned = AccuracyTarget::EpsilonDelta { + epsilon: 0.01, + delta: 0.001, + }; + let candidates = all_candidates("quantile_over_time(0.99, m[1m])", &planned); let strict = AccuracyTarget::EpsilonDelta { epsilon: 1e-6, delta: 1e-9, }; + let summaries: Vec<_> = candidates + .iter() + .filter(|c| has_summary_estimate(c)) + .map(|c| c.id.as_str()) + .collect(); + assert!(!summaries.is_empty()); let selection = select_plan( &candidates, &[Some(strict)], @@ -630,20 +641,17 @@ mod tests { PlanningModels::builtin(), ) .unwrap(); - assert_eq!(selection.selected, "P1"); - let rejected: BTreeMap<_, _> = selection - .rejected - .iter() - .map(|r| (r.id.as_str(), r)) - .collect(); - assert_eq!(rejected.len(), 2); - assert!(!rejected["P3"].valid); - assert!( - rejected["P3"].reason.contains("misses target"), - "{}", - rejected["P3"].reason - ); - assert!(!selection.costs.contains_key("P3")); + assert!(!summaries.contains(&selection.selected.as_str())); + for id in &summaries { + let rejection = selection.rejected.iter().find(|r| r.id == *id).unwrap(); + assert!(!rejection.valid); + assert!( + rejection.reason.contains("misses target"), + "{}", + rejection.reason + ); + assert!(!selection.costs.contains_key(*id)); + } } /// Count-Min over weights not proven non-negative is invalid whatever @@ -667,7 +675,9 @@ mod tests { } /// Each node is priced once, under its DAG id, and the total is the sum; - /// every candidate is either selected or rejected as costlier. + /// every candidate is either selected or rejected as costlier. The + /// CountSketch+heap estimate has no analytical guarantee, so deployment + /// evidence admits it. #[test] fn per_node_costs_cover_the_dag_and_sum_to_the_total() { let candidates = candidates(); @@ -679,7 +689,7 @@ mod tests { &candidates, &[Some(target)], &data(), - PlanningModels::builtin(), + PlanningModels::builtin().with_evidence(&EveryOutputMeasuredExact), ) .unwrap(); for candidate in candidates.iter().filter(|c| c.id != "P2") { From 251c746231a3bd7c0dc396ab4556518b47b91aa7 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 13 Oct 2026 04:59:01 +0000 Subject: [PATCH 11/17] fix(planner): exact top-k sorts by the ranked measure in physical optimization Rank a one-measure grouped child by its measure (the SQL heavy-hitter and PromQL `topk(k, sum by ..)` shapes) and per-series rows by the sample value, instead of looking up a `value` column. A node reading the top-k gets a Project restoring the logical [keys.., topk_k] columns over the rows. An Aggregate{[TopK]} with other measures, a filter or HAVING now fails, so its candidate is pruned with the reason. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W7qG9aFyPij5uWsyAJCxDW --- .../src/physical_asap_optimization.rs | 313 +++++++++++++++--- .../tests/physical_asap_optimization_sql.rs | 70 ++++ 2 files changed, 336 insertions(+), 47 deletions(-) create mode 100644 crates/integration-tests/tests/physical_asap_optimization_sql.rs diff --git a/crates/asap-aware-mapping/src/physical_asap_optimization.rs b/crates/asap-aware-mapping/src/physical_asap_optimization.rs index d009fae6..939486f9 100644 --- a/crates/asap-aware-mapping/src/physical_asap_optimization.rs +++ b/crates/asap-aware-mapping/src/physical_asap_optimization.rs @@ -13,6 +13,8 @@ //! top-k, which it cannot run as `Aggregate{[TopK]}`. Stage 2 records that //! choice in the DAG by rewriting it to a per-group sort followed by a //! per-group limit, the shape the PromQL frontend uses for generic `topk`. +//! An `Aggregate{[TopK]}` with other measures, a filter or `HAVING` has no +//! such rewrite, and its candidate fails. //! A summary needs no rewrite: `SummaryAgg` → `SummaryEstimate` already is //! build → estimate. Every node keeps the guarantee Stage 1 composed for it. use std::collections::HashMap; @@ -23,11 +25,12 @@ use asap_types::ir::physical_export::{ }; use asap_types::ir::{ apply_materialization_timings, MaterializationAssignment, NonASAPOp, Operator, OperatorNode, - QueryRoot, ScalarExpr, SchemaDerivationError, SortKey, TimingMemo, + ProjectItem, QueryRoot, ScalarExpr, SchemaDerivationError, SortKey, TimingMemo, }; use asap_types::post_asap::{ExecutionDataStateError, Materialization}; use asap_types::pre_asap::column_resolution::resolve_column_ref; use asap_types::pre_asap::expr_ir::ColumnRef; +use asap_types::pre_asap::schema::ColumnId; use asap_types::pre_asap::{AggIntent, Reduction}; use thiserror::Error; @@ -50,8 +53,8 @@ pub enum PhysicalASAPOptimizationError { Structure(#[from] SchemaDerivationError), #[error(transparent)] Timing(#[from] ExecutionDataStateError), - #[error("exact top-k has no sortable value column: {0}")] - NoValueColumn(String), + #[error("exact top-k has no Sort + Limit implementation: {0}")] + ExactTopK(String), } /// Stage 2 for one CandidateLogicalASAPDAG: implement every operator @@ -106,60 +109,138 @@ fn map_roots( .collect() } -type Memo = HashMap<*const OperatorNode, Rc>; +/// Each implemented node in two forms: as a query root, and as the input of +/// another node. They differ only for exact top-k (see [`implement_topk`]). +type Memo = HashMap<*const OperatorNode, (Rc, Rc)>; fn implement( node: &Rc, memo: &mut Memo, ) -> Result, PhysicalASAPOptimizationError> { - if let Some(done) = memo.get(&Rc::as_ptr(node)) { - return Ok(done.clone()); + if let Some((as_root, _)) = memo.get(&Rc::as_ptr(node)) { + return Ok(as_root.clone()); } for child in node.children() { implement(child, memo)?; } - let implemented = match node.non_asap() { + let forms = match node.non_asap() { + Some(NonASAPOp::Aggregate { measures, .. }) + if measures.iter().any(|m| matches!(m, AggIntent::TopK { .. })) => + { + implement_topk(node, memo)? + } + _ => { + let rebuilt = rebuilt(node, memo)?; + (rebuilt.clone(), rebuilt) + } + }; + let as_root = forms.0.clone(); + memo.insert(Rc::as_ptr(node), forms); + Ok(as_root) +} + +/// Exact top-k as `Limit(Sort)` per group, which outputs the ranked rows of +/// the child. That is the query result at a root; a node reading the top-k +/// instead resolved its columns against the logical `[keys.., topk_k]` +/// output, so it reads a `Project` restoring those names over the rows (the +/// ranked value under `topk_k`). +fn implement_topk( + node: &Rc, + memo: &Memo, +) -> Result<(Rc, Rc), PhysicalASAPOptimizationError> { + let unsupported = |why: &str| PhysicalASAPOptimizationError::ExactTopK(why.to_string()); + let Some(NonASAPOp::Aggregate { + reduction, + measures, + filters, + having, + child, + .. + }) = node.non_asap() + else { + unreachable!("implement_topk is called on an Aggregate") + }; + let Reduction::Reduce(keys) = reduction else { + return Err(unsupported("per-entity reduction")); + }; + let [AggIntent::TopK { k, .. }] = measures.as_slice() else { + return Err(unsupported("measures besides the top-k")); + }; + if filters.iter().any(Option::is_some) { + return Err(unsupported("filtered measure")); + } + if having.is_some() { + return Err(unsupported("HAVING")); + } + let child = memo[&Rc::as_ptr(child)].1.clone(); + let value = ranked_column(&child)?; + let sorted = Rc::new( + OperatorNode::new(Operator::NonASAP(NonASAPOp::Sort { + keys: vec![SortKey { + expr: ScalarExpr::Column(value), + ascending: false, + nulls_first: false, + }], + partition_by: keys.clone(), + child: child.clone(), + }))? + .with_guarantee(child.guarantee.clone()), + ); + let limited = Rc::new( + OperatorNode::new(Operator::NonASAP(NonASAPOp::Limit { + n: Some(*k), + offset: 0, + partition_by: keys.clone(), + child: sorted, + }))? + .with_guarantee(node.guarantee.clone()), + ); + let (ranked, key_fields) = node + .schema + .fields + .split_last() + .expect("an aggregate outputs its measure"); + let mut cols = key_fields + .iter() + .map(|field| { + let column = resolve_column_ref(&ColumnRef::Named(field.name.clone()), &child.schema) + .map_err(|e| unsupported(&format!("group key {}: {e}", field.name)))?; + Ok(ProjectItem { + alias: Some(field.name.clone()), + expr: ScalarExpr::Column(column), + }) + }) + .collect::, PhysicalASAPOptimizationError>>()?; + cols.push(ProjectItem { + alias: Some(ranked.name.clone()), + expr: ScalarExpr::Column(value), + }); + let as_input = Rc::new( + OperatorNode::new(Operator::NonASAP(NonASAPOp::Project { + cols, + qualifier: None, + child: limited.clone(), + }))? + .with_guarantee(node.guarantee.clone()), + ); + Ok((limited, as_input)) +} + +/// The column exact top-k ranks `child` by. Over a one-measure grouped +/// aggregate (the SQL heavy-hitter shape and PromQL `topk(k, sum by ..)`) it +/// is that measure, after the group keys; over per-series rows (PromQL +/// `topk by (..) (k, rows)`) it is the sample value. +fn ranked_column(child: &OperatorNode) -> Result { + match child.non_asap() { Some(NonASAPOp::Aggregate { - reduction: Reduction::Reduce(keys), + reduction: Reduction::Reduce(_), measures, - filters, - having: None, - child, .. - }) if filters.iter().all(Option::is_none) => match measures.as_slice() { - [AggIntent::TopK { k, .. }] => { - let child = memo[&Rc::as_ptr(child)].clone(); - let value = resolve_column_ref(&ColumnRef::SampleValue, &child.schema) - .map_err(|e| PhysicalASAPOptimizationError::NoValueColumn(e.to_string()))?; - let guarantee = child.guarantee.clone(); - let sorted = Rc::new( - OperatorNode::new(Operator::NonASAP(NonASAPOp::Sort { - keys: vec![SortKey { - expr: ScalarExpr::Column(value), - ascending: false, - nulls_first: false, - }], - partition_by: keys.clone(), - child, - }))? - .with_guarantee(guarantee), - ); - Rc::new( - OperatorNode::new(Operator::NonASAP(NonASAPOp::Limit { - n: Some(*k), - offset: 0, - partition_by: keys.clone(), - child: sorted, - }))? - .with_guarantee(node.guarantee.clone()), - ) - } - _ => rebuilt(node, memo)?, - }, - _ => rebuilt(node, memo)?, - }; - memo.insert(Rc::as_ptr(node), implemented.clone()); - Ok(implemented) + }) if measures.len() == 1 => Ok(child.schema.fields.len() - 1), + _ => resolve_column_ref(&ColumnRef::SampleValue, &child.schema).map_err(|e| { + PhysicalASAPOptimizationError::ExactTopK(format!("no ranked column: {e}")) + }), + } } /// `node` over its implemented children; the same `Rc` when none changed. @@ -167,13 +248,14 @@ fn rebuilt( node: &Rc, memo: &Memo, ) -> Result, PhysicalASAPOptimizationError> { + let input = |child: &Rc| memo[&Rc::as_ptr(child)].1.clone(); let changed = node .children() .iter() - .any(|child| !Rc::ptr_eq(child, &memo[&Rc::as_ptr(child)])); + .any(|child| !Rc::ptr_eq(child, &input(child))); Ok(if changed { Rc::new( - node.with_new_children(|child| memo[&Rc::as_ptr(child)].clone())? + node.with_new_children(|child| input(child))? .with_guarantee(node.guarantee.clone()), ) } else { @@ -191,6 +273,143 @@ mod tests { use asap_types::post_asap::ExecutionTiming; use asap_types::types::AccuracyTarget; + fn operator_root(candidate: &PhysicalCandidate, i: usize) -> Rc { + match &candidate.roots[i] { + QueryRoot::Operator(node) => node.clone(), + QueryRoot::Scalar(_) => panic!("operator root"), + } + } + + /// The single sort key of the `Sort` under the `Limit` at `limit`. + fn topk_sort_key(limit: &OperatorNode) -> (ScalarExpr, bool) { + let NonASAPOp::Limit { child: sort, .. } = limit.expect_non_asap() else { + panic!("limit, found {}", limit.operator.kind_name()) + }; + let NonASAPOp::Sort { keys, .. } = sort.expect_non_asap() else { + panic!("sort below limit") + }; + let [key] = keys.as_slice() else { + panic!("one sort key") + }; + (key.expr.clone(), key.ascending) + } + + /// Example 1's shape, `topk by (job) (k, per-series rows)`, sorts each + /// group by the sample value, descending. + #[test] + fn promql_topk_over_series_sorts_by_value_descending() { + let root = lower_promql( + "topk by (job) (10, sum_over_time(m[1m]))", + AccuracyTarget::Exact, + ); + let candidate = optimize_physical_asap("L1", &[root.into()]).unwrap(); + let limit = operator_root(&candidate, 0); + assert_eq!(limit.schema.fields[1].name, "value"); + assert_eq!(topk_sort_key(&limit), (ScalarExpr::Column(1), false)); + } + + /// `topk(k, sum by (job) (..))` sorts by the summed measure. + #[test] + fn promql_topk_over_grouped_aggregate_sorts_by_its_measure() { + let root = lower_promql("topk(5, sum by (job) (m))", AccuracyTarget::Exact); + let candidate = optimize_physical_asap("L1", &[root.into()]).unwrap(); + let limit = operator_root(&candidate, 0); + let names: Vec<_> = limit + .schema + .fields + .iter() + .map(|f| f.name.as_str()) + .collect(); + assert_eq!(names, ["job", "sum"]); + assert_eq!(topk_sort_key(&limit), (ScalarExpr::Column(1), false)); + } + + /// A node reading an exact top-k keeps resolving the columns it bound + /// against the logical `[keys.., topk_k]` output: `sum by (job)` still + /// groups by `job`, not by whatever column now sits at its index. + #[test] + fn consumer_of_exact_topk_reads_its_logical_columns() { + let root = lower_promql( + "sum by (job) (topk by (job) (10, sum_over_time(m[1m])))", + AccuracyTarget::Exact, + ); + let candidate = optimize_physical_asap("L1", &[root.clone().into()]).unwrap(); + let physical = operator_root(&candidate, 0); + let names = |node: &OperatorNode| -> Vec { + node.schema.fields.iter().map(|f| f.name.clone()).collect() + }; + assert_eq!(names(&physical), names(&root)); + let NonASAPOp::Aggregate { child: input, .. } = physical.expect_non_asap() else { + panic!("sum root") + }; + let NonASAPOp::Project { + cols, child: limit, .. + } = input.expect_non_asap() + else { + panic!("project over the top-k rows") + }; + let read: Vec<_> = cols + .iter() + .map(|item| { + let ScalarExpr::Column(c) = item.expr else { + panic!("column") + }; + ( + item.alias.clone().unwrap(), + limit.schema.fields[c].name.clone(), + ) + }) + .collect(); + assert_eq!( + read, + [ + ("job".to_string(), "job".to_string()), + ("topk_10".to_string(), "value".to_string()) + ] + ); + assert_eq!(topk_sort_key(limit), (ScalarExpr::Column(1), false)); + } + + /// An `Aggregate{[TopK]}` that Sort + Limit cannot express (another + /// measure, `HAVING`) fails, so its candidate is pruned with the reason. + #[test] + fn topk_with_extra_measure_or_having_fails() { + use crate::test_support::{aggregate, metric_scan}; + use asap_types::ir::Predicate; + use asap_types::pre_asap::expr_ir::ScalarValue; + let topk = AggIntent::TopK { + k: 3, + accuracy: AccuracyTarget::Exact, + }; + let count = AggIntent::Count { + accuracy: AccuracyTarget::Exact, + }; + let by_job = || Reduction::by(vec![2]); + let cases = [ + aggregate( + by_job(), + vec![topk.clone(), count], + vec![], + None, + metric_scan(&["job"]), + ), + aggregate( + by_job(), + vec![topk], + vec![], + Some(Predicate(ScalarExpr::Literal(ScalarValue::Boolean(true)))), + metric_scan(&["job"]), + ), + ]; + for root in cases { + let error = optimize_physical_asap("L1", &[root.into()]).unwrap_err(); + assert!( + matches!(error, PhysicalASAPOptimizationError::ExactTopK(_)), + "{error}" + ); + } + } + /// Exact `topk by (job)` becomes Limit(Sort) partitioned by `job`, and a /// child shared by two roots stays one node (one `Rc`, one DAG node). #[test] diff --git a/crates/integration-tests/tests/physical_asap_optimization_sql.rs b/crates/integration-tests/tests/physical_asap_optimization_sql.rs new file mode 100644 index 00000000..4491296a --- /dev/null +++ b/crates/integration-tests/tests/physical_asap_optimization_sql.rs @@ -0,0 +1,70 @@ +//! SQL heavy-hitter queries through physical ASAP-aware optimization: exact +//! top-k sorts by the ranked count, whatever the group key is. + +use asap_aware_mapping::physical_asap_optimization::optimize_physical_asap; +use asap_frontend_sql::{lower_sql, SqlCatalog}; +use asap_types::ir::{NonASAPOp, QueryRoot, ScalarExpr}; +use asap_types::pre_asap::schema::{DataType, Field, Schema}; +use asap_types::types::AccuracyTarget; + +/// The root's output column names and its `(sort column name, ascending)`. +async fn exact_topk(sql: &str) -> (Vec, (String, bool)) { + let catalog = SqlCatalog::new().with_table( + "metrics", + Schema::new(vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + Field::plain("port", DataType::Int64, false), + ]), + ); + let root = lower_sql(sql, &catalog, AccuracyTarget::Exact) + .await + .unwrap(); + let candidate = optimize_physical_asap("L1", &[QueryRoot::Operator(root)]).unwrap(); + let QueryRoot::Operator(limit) = &candidate.roots[0] else { + panic!("operator root") + }; + let NonASAPOp::Limit { + n: Some(5), + child: sort, + .. + } = limit.expect_non_asap() + else { + panic!("limit root") + }; + let NonASAPOp::Sort { keys, .. } = sort.expect_non_asap() else { + panic!("sort below limit") + }; + let [key] = keys.as_slice() else { + panic!("one sort key") + }; + let ScalarExpr::Column(column) = key.expr else { + panic!("sort by a column") + }; + let names = limit.schema.fields.iter().map(|f| f.name.clone()).collect(); + ( + names, + (sort.schema.fields[column].name.clone(), key.ascending), + ) +} + +/// Grouping by a column named `value` still ranks by the count, descending. +#[tokio::test] +async fn group_key_named_value_ranks_by_count() { + let (names, sort) = exact_topk( + "SELECT value, COUNT(*) AS c FROM metrics GROUP BY value ORDER BY c DESC LIMIT 5", + ) + .await; + assert_eq!(names, ["value", "count(Int64(1))"]); + assert_eq!(sort, ("count(Int64(1))".to_string(), false)); +} + +/// An integer group key leaves no `value` column; the count is still ranked. +#[tokio::test] +async fn integer_group_key_ranks_by_count() { + let (names, sort) = + exact_topk("SELECT port, COUNT(*) AS c FROM metrics GROUP BY port ORDER BY c DESC LIMIT 5") + .await; + assert_eq!(names, ["port", "count(Int64(1))"]); + assert_eq!(sort, ("count(Int64(1))".to_string(), false)); +} From 754136ca09ae676a2ad86dc25e6cb0cf5b658213 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 13 Oct 2026 04:59:52 +0000 Subject: [PATCH 12/17] fix(devtools): stage_pipeline emits plan_selection when selection fails With no valid candidate the document lists the rejections with a null `selected` (and the Stage 2 pruned list as before); any other selection error is reported under `error`, instead of failing the run. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W7qG9aFyPij5uWsyAJCxDW --- crates/devtools/src/bin/stage_pipeline.rs | 70 ++++++++++++++++++++--- 1 file changed, 61 insertions(+), 9 deletions(-) diff --git a/crates/devtools/src/bin/stage_pipeline.rs b/crates/devtools/src/bin/stage_pipeline.rs index 81175386..d843fa28 100644 --- a/crates/devtools/src/bin/stage_pipeline.rs +++ b/crates/devtools/src/bin/stage_pipeline.rs @@ -16,7 +16,9 @@ // cost; a composed logical candidate Stage 2 cannot implement is listed // under `pruned` with the reason; // - plan_selection: per-candidate costs, the selected candidate, and -// every other candidate as rejected (`valid: false`) or costlier. +// every other candidate as rejected (`valid: false`) or costlier. When +// selection fails, `selected` is null, with the rejections, or with +// `error` for any other failure. // // `--promql` may repeat. `--epsilon`/`--delta` apply to every `--promql` // query; without them the queries are exact. `--interval-ms` is the source @@ -29,7 +31,7 @@ use asap_aware_mapping::per_computation_candidates::{ PerComputationCandidates, }; use asap_aware_mapping::physical_asap_optimization::optimize_physical_asap; -use asap_aware_mapping::plan_selection::{select_plan, Selection}; +use asap_aware_mapping::plan_selection::{select_plan, Rejection, Selection, SelectionError}; use asap_aware_mapping::PlanningModels; use asap_types::ir::flat::{flatten, FlatDag}; use asap_types::ir::schema_support::with_promql_series_identity; @@ -162,8 +164,7 @@ fn stage_pipeline(workload: &PlanningWorkload, max_candidates: usize) -> Result< .map(|entry| Some(entry.requirements.accuracy.target())) .collect(); let data = workload.data_workload.clone().unwrap_or_default(); - let selection = select_plan(&physical, &targets, &data, PlanningModels::builtin()) - .map_err(|e| format!("Stage 3: {e}"))?; + let selection = select_plan(&physical, &targets, &data, PlanningModels::builtin()); let physical_json: Vec<_> = physical .iter() .map(|p| json!({ "id": p.id, "from_logical": p.from_logical, "label": p.label, "dag": p.dag })) @@ -182,7 +183,16 @@ fn stage_pipeline(workload: &PlanningWorkload, max_candidates: usize) -> Result< })) } -fn plan_selection_json(selection: &Selection) -> Value { +fn plan_selection_json(selection: &Result) -> Value { + let selection = match selection { + Ok(selection) => selection, + Err(SelectionError::NoValidCandidate(rejected)) => { + return json!({ "costs": {}, "selected": null, "rejected": rejections_json(rejected) }) + } + Err(error) => { + return json!({ "costs": {}, "selected": null, "rejected": [], "error": error.to_string() }) + } + }; let costs: serde_json::Map<_, _> = selection .costs .iter() @@ -203,12 +213,18 @@ fn plan_selection_json(selection: &Selection) -> Value { ) }) .collect(); - let rejected: Vec<_> = selection - .rejected + json!({ + "costs": costs, + "selected": selection.selected, + "rejected": rejections_json(&selection.rejected), + }) +} + +fn rejections_json(rejected: &[Rejection]) -> Vec { + rejected .iter() .map(|r| json!({ "id": r.id, "valid": r.valid, "reason": r.reason })) - .collect(); - json!({ "costs": costs, "selected": selection.selected, "rejected": rejected }) + .collect() } /// A logical DAG as a flat node list: node `i` is `nodes[i]`, and children are @@ -412,3 +428,39 @@ fn planner_layering_example1() -> PlanningWorkload { }), } } + +#[cfg(test)] +mod tests { + use super::*; + + /// With no valid candidate, the document is still written: `selected` + /// is null and every rejection is listed. + #[test] + fn failed_selection_lists_rejections_without_a_selection() { + let rejected = vec![Rejection { + id: "P1".into(), + valid: false, + reason: "q1: misses target".into(), + }]; + let document = plan_selection_json(&Err(SelectionError::NoValidCandidate(rejected))); + assert_eq!( + document, + json!({ + "costs": {}, + "selected": null, + "rejected": [{ "id": "P1", "valid": false, "reason": "q1: misses target" }], + }) + ); + } + + /// A run with no physical candidate (here, none enumerated) still emits + /// every stage, with no selection. + #[test] + fn run_without_candidates_emits_plan_selection() { + let workload = promql_batch(&["sum(m)".into()], AccuracyTarget::Exact, 15_000); + let document = stage_pipeline(&workload, 0).unwrap(); + assert_eq!(document["physical_asap_optimization"]["pruned"], json!([])); + assert_eq!(document["plan_selection"]["selected"], Value::Null); + assert_eq!(document["plan_selection"]["rejected"], json!([])); + } +} From eed8693b3daec4375cfe8e7e306d45c49e76564b Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 13 Oct 2026 05:00:13 +0000 Subject: [PATCH 13/17] docs(planner): plan selection checks Count-Min only under a target; note MVP costing limits Reword the Count-Min test to the decided behavior and test that a query without a target admits the Count-Min candidate. Document that a cost error aborts selection (only from bad declared rates) and that binary, join and concat nodes are priced from their first input only. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W7qG9aFyPij5uWsyAJCxDW --- .../asap-aware-mapping/src/plan_selection.rs | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/crates/asap-aware-mapping/src/plan_selection.rs b/crates/asap-aware-mapping/src/plan_selection.rs index 8ffeab97..6cce4db1 100644 --- a/crates/asap-aware-mapping/src/plan_selection.rs +++ b/crates/asap-aware-mapping/src/plan_selection.rs @@ -5,7 +5,8 @@ //! guarantee does (theoretical: Stage 1 composes it with the built-in //! accuracy model), or when the deployment's empirical evidence shows this //! candidate's output does (`AccuracyEvidenceProvider::output_guarantee`, -//! read only here). Count-Min is admitted only over weights proven non-negative. +//! read only here). For a query with a target, Count-Min is admitted only +//! over weights proven non-negative; a query without one constrains nothing. //! A miss rejects the candidate as invalid, with a reason; only this stage //! discards valid candidates. //! Every valid candidate is priced node by node over its physical DAG, so a @@ -15,8 +16,11 @@ //! Prices come from [`crate::analytical_cost::estimate_operator`] over edge //! statistics derived from the [`DataWorkload`] and a fixed default group //! count; summary build and estimation are priced as rows × sketch depth and -//! groups × k. These numbers are illustrative, not calibrated. Latency -//! bounds and deployment capabilities are not checked yet. +//! groups × k. These numbers are illustrative, not calibrated: a binary, +//! join or concat node is priced from its first input only (MVP; follow-up). +//! A cost error on any candidate aborts selection; it is reachable only from +//! bad declared rates. Latency bounds and deployment capabilities are not +//! checked yet. use std::collections::{BTreeMap, HashMap}; use std::rc::Rc; @@ -654,8 +658,8 @@ mod tests { } } - /// Count-Min over weights not proven non-negative is invalid whatever - /// the target. + /// For a query with a target, Count-Min over weights not proven + /// non-negative is invalid. #[test] fn count_min_over_signed_weights_is_rejected_as_invalid() { let target = AccuracyTarget::EpsilonDelta { @@ -674,6 +678,16 @@ mod tests { assert!(p2.reason.contains("non-negative"), "{}", p2.reason); } + /// A query without a target constrains nothing: the Count-Min + /// candidate is admitted and priced. + #[test] + fn count_min_is_admitted_without_a_target() { + let selection = + select_plan(&candidates(), &[None], &data(), PlanningModels::builtin()).unwrap(); + assert!(selection.costs.contains_key("P2")); + assert!(selection.rejected.iter().all(|r| r.valid)); + } + /// Each node is priced once, under its DAG id, and the total is the sum; /// every candidate is either selected or rejected as costlier. The /// CountSketch+heap estimate has no analytical guarantee, so deployment From 03b07e65750632a0c60b9029cbb529c4da364a3e Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:57:52 +0000 Subject: [PATCH 14/17] fix(planner): exact top-k ranks by the measure of a composed Stage 1 child After Pass 1 composition a top-k reads its one-measure aggregate as Project(FinalizeExactAccumulator(SummaryAgg)) or Project(SummaryEstimate(..)), not as an Aggregate, so ranking fell back to the sample value: a key named `value` was sorted on, and an integer or multi-column key pruned the candidate. Look through renaming Projects, Filters, finalize and estimate to the SummaryAgg or Aggregate; a grouped one ranks by its last column. The SQL test now composes every Pass 1 choice under exact and approximate targets. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W7qG9aFyPij5uWsyAJCxDW --- .../src/physical_asap_optimization.rs | 49 ++++- .../tests/physical_asap_optimization_sql.rs | 175 ++++++++++++++---- 2 files changed, 178 insertions(+), 46 deletions(-) diff --git a/crates/asap-aware-mapping/src/physical_asap_optimization.rs b/crates/asap-aware-mapping/src/physical_asap_optimization.rs index 939486f9..e1e4dd34 100644 --- a/crates/asap-aware-mapping/src/physical_asap_optimization.rs +++ b/crates/asap-aware-mapping/src/physical_asap_optimization.rs @@ -24,8 +24,8 @@ use asap_types::ir::physical_export::{ compile_physical_asap_workload_with_node_ids, PhysicalASAPDAG, }; use asap_types::ir::{ - apply_materialization_timings, MaterializationAssignment, NonASAPOp, Operator, OperatorNode, - ProjectItem, QueryRoot, ScalarExpr, SchemaDerivationError, SortKey, TimingMemo, + apply_materialization_timings, ASAPOp, MaterializationAssignment, NonASAPOp, Operator, + OperatorNode, ProjectItem, QueryRoot, ScalarExpr, SchemaDerivationError, SortKey, TimingMemo, }; use asap_types::post_asap::{ExecutionDataStateError, Materialization}; use asap_types::pre_asap::column_resolution::resolve_column_ref; @@ -228,15 +228,44 @@ fn implement_topk( /// The column exact top-k ranks `child` by. Over a one-measure grouped /// aggregate (the SQL heavy-hitter shape and PromQL `topk(k, sum by ..)`) it -/// is that measure, after the group keys; over per-series rows (PromQL -/// `topk by (..) (k, rows)`) it is the sample value. +/// is that measure, the last column after the group keys, whichever Stage 1 +/// candidate realizes it: the aggregate itself, or a `SummaryAgg` read by +/// `FinalizeExactAccumulator` or `SummaryEstimate`, under the `Project` +/// restoring its names (a renaming) and a value `HAVING`'s `Filter`. Over per-series rows +/// (PromQL `topk by (..) (k, rows)`) it is the sample value. fn ranked_column(child: &OperatorNode) -> Result { - match child.non_asap() { - Some(NonASAPOp::Aggregate { - reduction: Reduction::Reduce(_), - measures, - .. - }) if measures.len() == 1 => Ok(child.schema.fields.len() - 1), + let mut node = child; + let reduction = loop { + match &node.operator { + // Only a renaming keeps the measure last. + Operator::NonASAP(NonASAPOp::Project { cols, child, .. }) + if cols.len() == child.schema.fields.len() + && cols + .iter() + .enumerate() + .all(|(i, item)| item.expr == ScalarExpr::Column(i)) => + { + node = child + } + Operator::NonASAP(NonASAPOp::Filter { child, .. }) + | Operator::ASAP( + ASAPOp::FinalizeExactAccumulator { child } + | ASAPOp::SummaryEstimate { + summary_input: child, + .. + }, + ) => node = child, + Operator::ASAP(ASAPOp::SummaryAgg { reduction, .. }) => break Some(reduction), + Operator::NonASAP(NonASAPOp::Aggregate { + reduction, + measures, + .. + }) if measures.len() == 1 => break Some(reduction), + _ => break None, + } + }; + match reduction { + Some(Reduction::Reduce(_)) => Ok(child.schema.fields.len() - 1), _ => resolve_column_ref(&ColumnRef::SampleValue, &child.schema).map_err(|e| { PhysicalASAPOptimizationError::ExactTopK(format!("no ranked column: {e}")) }), diff --git a/crates/integration-tests/tests/physical_asap_optimization_sql.rs b/crates/integration-tests/tests/physical_asap_optimization_sql.rs index 4491296a..c593a1e9 100644 --- a/crates/integration-tests/tests/physical_asap_optimization_sql.rs +++ b/crates/integration-tests/tests/physical_asap_optimization_sql.rs @@ -1,36 +1,36 @@ -//! SQL heavy-hitter queries through physical ASAP-aware optimization: exact -//! top-k sorts by the ranked count, whatever the group key is. +//! SQL heavy-hitter queries through Stage 1 (every Pass 1 choice composed) +//! and physical ASAP-aware optimization: exact top-k sorts by the ranked +//! measure, descending, whatever the group keys are called or typed, and no +//! valid candidate is pruned. +use asap_aware_mapping::per_computation_candidates::{ + compose_logical_candidate, generate_per_computation_candidates, ComputationCandidate, +}; use asap_aware_mapping::physical_asap_optimization::optimize_physical_asap; use asap_frontend_sql::{lower_sql, SqlCatalog}; -use asap_types::ir::{NonASAPOp, QueryRoot, ScalarExpr}; +use asap_types::ir::{NonASAPOp, OperatorNode, QueryRoot, ScalarExpr}; use asap_types::pre_asap::schema::{DataType, Field, Schema}; +use asap_types::pre_asap::AggIntent; use asap_types::types::AccuracyTarget; -/// The root's output column names and its `(sort column name, ascending)`. -async fn exact_topk(sql: &str) -> (Vec, (String, bool)) { - let catalog = SqlCatalog::new().with_table( - "metrics", - Schema::new(vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - Field::plain("port", DataType::Int64, false), - ]), - ); - let root = lower_sql(sql, &catalog, AccuracyTarget::Exact) - .await - .unwrap(); - let candidate = optimize_physical_asap("L1", &[QueryRoot::Operator(root)]).unwrap(); - let QueryRoot::Operator(limit) = &candidate.roots[0] else { - panic!("operator root") - }; +fn approximate() -> AccuracyTarget { + AccuracyTarget::EpsilonDelta { + epsilon: 0.01, + delta: 0.001, + } +} + +/// The `(sort column name, ascending)` of an exact top-k's `Sort`, if `root` +/// is one (a heavy-hitter summary candidate has none). +fn exact_topk_sort(root: &OperatorNode) -> Option<(String, bool)> { + let limit = root; let NonASAPOp::Limit { n: Some(5), child: sort, .. - } = limit.expect_non_asap() + } = limit.non_asap()? else { - panic!("limit root") + return None; }; let NonASAPOp::Sort { keys, .. } = sort.expect_non_asap() else { panic!("sort below limit") @@ -41,30 +41,133 @@ async fn exact_topk(sql: &str) -> (Vec, (String, bool)) { let ScalarExpr::Column(column) = key.expr else { panic!("sort by a column") }; - let names = limit.schema.fields.iter().map(|f| f.name.clone()).collect(); - ( - names, - (sort.schema.fields[column].name.clone(), key.ascending), - ) + Some((sort.schema.fields[column].name.clone(), key.ascending)) +} + +/// Every Pass 1 choice of `sql` under `accuracy`, composed and physically +/// optimized: each must succeed, every exact top-k must sort by `ranked` +/// descending. Returns how many +/// candidates had an exact top-k. +async fn check_every_choice(sql: &str, accuracy: AccuracyTarget, ranked: &str) -> usize { + let catalog = SqlCatalog::new().with_table( + "metrics", + Schema::new(vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + Field::plain("host", DataType::Utf8, false), + Field::plain("port", DataType::Int64, false), + ]), + ); + let root = lower_sql(sql, &catalog, accuracy).await.unwrap(); + let NonASAPOp::Aggregate { measures, .. } = root.expect_non_asap() else { + panic!("{sql}: heavy-hitter aggregate root") + }; + assert!( + matches!(measures.as_slice(), [AggIntent::TopK { k: 5, .. }]), + "{sql}" + ); + let inventory = + generate_per_computation_candidates(vec![(0, QueryRoot::Operator(root))]).unwrap(); + // The cartesian product of every target's candidates. + let mut choices = vec![vec![]]; + for target in &inventory.targets { + choices = choices + .into_iter() + .flat_map(|prefix: Vec| { + (0..target.choices.candidates.len()).map(move |i| { + let mut choice = prefix.clone(); + choice.push(i); + choice + }) + }) + .collect(); + } + let mut exact = 0; + for choice in choices { + let composed = match compose_logical_candidate(&inventory, &choice) { + Ok(composed) => composed, + // Stage 1 cannot yet compose a heavy-hitter summary over some + // keys (an integer or several key columns); Stage 2 never sees + // that candidate. Every other choice must compose. + Err(e) => { + let summarized_topk = inventory.targets.iter().zip(&choice).any(|(t, &i)| { + let NonASAPOp::Aggregate { measures, .. } = t.target.expect_non_asap() else { + return false; + }; + matches!(measures[t.measure], AggIntent::TopK { .. }) + && t.choices.candidates[i] != ComputationCandidate::Exact + }); + assert!(summarized_topk, "{sql}, choice {choice:?}: {e}"); + continue; + } + }; + let roots: Vec<_> = composed.into_iter().map(|(_, root)| root).collect(); + let candidate = optimize_physical_asap("L1", &roots) + .unwrap_or_else(|e| panic!("{sql}, choice {choice:?} pruned: {e}")); + let QueryRoot::Operator(physical) = &candidate.roots[0] else { + panic!("operator root") + }; + if let Some(sort) = exact_topk_sort(physical) { + assert_eq!( + sort, + (ranked.to_string(), false), + "{sql}, choice {choice:?}" + ); + exact += 1; + } + } + exact } -/// Grouping by a column named `value` still ranks by the count, descending. +/// Each query under an exact and an approximate target; every exact top-k +/// sorts by the ranked measure. +async fn check(sql: &str, ranked: &str) { + for accuracy in [AccuracyTarget::Exact, approximate()] { + let exact = check_every_choice(sql, accuracy.clone(), ranked).await; + assert!( + exact > 0, + "{sql} under {accuracy:?}: an exact top-k candidate" + ); + } +} + +/// Grouping by a column named `value` still ranks by the count. #[tokio::test] async fn group_key_named_value_ranks_by_count() { - let (names, sort) = exact_topk( + check( "SELECT value, COUNT(*) AS c FROM metrics GROUP BY value ORDER BY c DESC LIMIT 5", + "count(Int64(1))", ) .await; - assert_eq!(names, ["value", "count(Int64(1))"]); - assert_eq!(sort, ("count(Int64(1))".to_string(), false)); } /// An integer group key leaves no `value` column; the count is still ranked. #[tokio::test] async fn integer_group_key_ranks_by_count() { - let (names, sort) = - exact_topk("SELECT port, COUNT(*) AS c FROM metrics GROUP BY port ORDER BY c DESC LIMIT 5") - .await; - assert_eq!(names, ["port", "count(Int64(1))"]); - assert_eq!(sort, ("count(Int64(1))".to_string(), false)); + check( + "SELECT port, COUNT(*) AS c FROM metrics GROUP BY port ORDER BY c DESC LIMIT 5", + "count(Int64(1))", + ) + .await; +} + +/// Two group keys, one of them an integer. +#[tokio::test] +async fn multi_key_ranks_by_count() { + check( + "SELECT host, port, COUNT(*) AS c FROM metrics GROUP BY host, port \ + ORDER BY c DESC LIMIT 5", + "count(Int64(1))", + ) + .await; +} + +/// A SUM ranking sorts by the sum. +#[tokio::test] +async fn sum_ranking_ranks_by_sum() { + check( + "SELECT port, SUM(value) AS s FROM metrics GROUP BY port ORDER BY s DESC LIMIT 5", + "sum(metrics.value)", + ) + .await; } From 60197da25d2d894a1b210f60573bcfe99c6cc81f Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:58:24 +0000 Subject: [PATCH 15/17] fix(planner): exact top-k roots output the logical [keys.., topk_k] columns An exact top-k root returned its child's raw rows (Example 1's P1 leaked `ts`, `value` and `$promql_series_identity`), while summary candidates and the frontend give `[keys.., topk_k]`. The restoring Project now sits over the Limit at a root too, so one implementation serves roots and consumers. `topk_k` carries the ranked value's type and nullability, not the frontend's Utf8 placeholder. Regenerate the Example 1 fixture: P1 gains the root Project (node 12). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W7qG9aFyPij5uWsyAJCxDW --- .../src/physical_asap_optimization.rs | 94 ++++++------- .../tests/physical_asap_optimization_sql.rs | 26 +++- .../examples/planner-layering-example1.json | 126 +++++++++++++++++- 3 files changed, 194 insertions(+), 52 deletions(-) diff --git a/crates/asap-aware-mapping/src/physical_asap_optimization.rs b/crates/asap-aware-mapping/src/physical_asap_optimization.rs index e1e4dd34..3a748dee 100644 --- a/crates/asap-aware-mapping/src/physical_asap_optimization.rs +++ b/crates/asap-aware-mapping/src/physical_asap_optimization.rs @@ -13,6 +13,12 @@ //! top-k, which it cannot run as `Aggregate{[TopK]}`. Stage 2 records that //! choice in the DAG by rewriting it to a per-group sort followed by a //! per-group limit, the shape the PromQL frontend uses for generic `topk`. +//! A `Project` over the limited rows restores the logical `[keys.., topk_k]` +//! columns, so an exact root has the same column names as a summary +//! candidate's and as the frontend declares, and no input column leaks. +//! `topk_k` carries the ranked value's type and nullability (the frontend +//! declares a non-null `Utf8` placeholder); casting the value to `Utf8` would +//! not make it the same answer. //! An `Aggregate{[TopK]}` with other measures, a filter or `HAVING` has no //! such rewrite, and its candidate fails. //! A summary needs no rewrite: `SummaryAgg` → `SummaryEstimate` already is @@ -109,45 +115,39 @@ fn map_roots( .collect() } -/// Each implemented node in two forms: as a query root, and as the input of -/// another node. They differ only for exact top-k (see [`implement_topk`]). -type Memo = HashMap<*const OperatorNode, (Rc, Rc)>; +/// Each node's implementation. +type Memo = HashMap<*const OperatorNode, Rc>; fn implement( node: &Rc, memo: &mut Memo, ) -> Result, PhysicalASAPOptimizationError> { - if let Some((as_root, _)) = memo.get(&Rc::as_ptr(node)) { - return Ok(as_root.clone()); + if let Some(done) = memo.get(&Rc::as_ptr(node)) { + return Ok(done.clone()); } for child in node.children() { implement(child, memo)?; } - let forms = match node.non_asap() { + let implemented = match node.non_asap() { Some(NonASAPOp::Aggregate { measures, .. }) if measures.iter().any(|m| matches!(m, AggIntent::TopK { .. })) => { implement_topk(node, memo)? } - _ => { - let rebuilt = rebuilt(node, memo)?; - (rebuilt.clone(), rebuilt) - } + _ => rebuilt(node, memo)?, }; - let as_root = forms.0.clone(); - memo.insert(Rc::as_ptr(node), forms); - Ok(as_root) + memo.insert(Rc::as_ptr(node), implemented.clone()); + Ok(implemented) } /// Exact top-k as `Limit(Sort)` per group, which outputs the ranked rows of -/// the child. That is the query result at a root; a node reading the top-k -/// instead resolved its columns against the logical `[keys.., topk_k]` -/// output, so it reads a `Project` restoring those names over the rows (the -/// ranked value under `topk_k`). +/// the child, under a `Project` restoring the logical `[keys.., topk_k]` +/// output (the ranked value under `topk_k`): a root returns those columns, +/// and a node reading the top-k resolved its columns against them. fn implement_topk( node: &Rc, memo: &Memo, -) -> Result<(Rc, Rc), PhysicalASAPOptimizationError> { +) -> Result, PhysicalASAPOptimizationError> { let unsupported = |why: &str| PhysicalASAPOptimizationError::ExactTopK(why.to_string()); let Some(NonASAPOp::Aggregate { reduction, @@ -172,7 +172,7 @@ fn implement_topk( if having.is_some() { return Err(unsupported("HAVING")); } - let child = memo[&Rc::as_ptr(child)].1.clone(); + let child = memo[&Rc::as_ptr(child)].clone(); let value = ranked_column(&child)?; let sorted = Rc::new( OperatorNode::new(Operator::NonASAP(NonASAPOp::Sort { @@ -215,15 +215,14 @@ fn implement_topk( alias: Some(ranked.name.clone()), expr: ScalarExpr::Column(value), }); - let as_input = Rc::new( + Ok(Rc::new( OperatorNode::new(Operator::NonASAP(NonASAPOp::Project { cols, qualifier: None, - child: limited.clone(), + child: limited, }))? .with_guarantee(node.guarantee.clone()), - ); - Ok((limited, as_input)) + )) } /// The column exact top-k ranks `child` by. Over a one-measure grouped @@ -277,7 +276,7 @@ fn rebuilt( node: &Rc, memo: &Memo, ) -> Result, PhysicalASAPOptimizationError> { - let input = |child: &Rc| memo[&Rc::as_ptr(child)].1.clone(); + let input = |child: &Rc| memo[&Rc::as_ptr(child)].clone(); let changed = node .children() .iter() @@ -309,6 +308,18 @@ mod tests { } } + fn names(node: &OperatorNode) -> Vec { + node.schema.fields.iter().map(|f| f.name.clone()).collect() + } + + /// The `Limit` under an exact top-k's restoring `Project`. + fn topk_limit(project: &OperatorNode) -> Rc { + let NonASAPOp::Project { child, .. } = project.expect_non_asap() else { + panic!("project, found {}", project.operator.kind_name()) + }; + child.clone() + } + /// The single sort key of the `Sort` under the `Limit` at `limit`. fn topk_sort_key(limit: &OperatorNode) -> (ScalarExpr, bool) { let NonASAPOp::Limit { child: sort, .. } = limit.expect_non_asap() else { @@ -331,8 +342,10 @@ mod tests { "topk by (job) (10, sum_over_time(m[1m]))", AccuracyTarget::Exact, ); - let candidate = optimize_physical_asap("L1", &[root.into()]).unwrap(); - let limit = operator_root(&candidate, 0); + let candidate = optimize_physical_asap("L1", &[root.clone().into()]).unwrap(); + let physical = operator_root(&candidate, 0); + assert_eq!(names(&physical), names(&root)); + let limit = topk_limit(&physical); assert_eq!(limit.schema.fields[1].name, "value"); assert_eq!(topk_sort_key(&limit), (ScalarExpr::Column(1), false)); } @@ -342,14 +355,10 @@ mod tests { fn promql_topk_over_grouped_aggregate_sorts_by_its_measure() { let root = lower_promql("topk(5, sum by (job) (m))", AccuracyTarget::Exact); let candidate = optimize_physical_asap("L1", &[root.into()]).unwrap(); - let limit = operator_root(&candidate, 0); - let names: Vec<_> = limit - .schema - .fields - .iter() - .map(|f| f.name.as_str()) - .collect(); - assert_eq!(names, ["job", "sum"]); + let physical = operator_root(&candidate, 0); + assert_eq!(names(&physical), ["topk_5"]); + let limit = topk_limit(&physical); + assert_eq!(names(&limit), ["job", "sum"]); assert_eq!(topk_sort_key(&limit), (ScalarExpr::Column(1), false)); } @@ -364,9 +373,6 @@ mod tests { ); let candidate = optimize_physical_asap("L1", &[root.clone().into()]).unwrap(); let physical = operator_root(&candidate, 0); - let names = |node: &OperatorNode| -> Vec { - node.schema.fields.iter().map(|f| f.name.clone()).collect() - }; assert_eq!(names(&physical), names(&root)); let NonASAPOp::Aggregate { child: input, .. } = physical.expect_non_asap() else { panic!("sum root") @@ -462,9 +468,9 @@ mod tests { partition_by, child: sort, .. - } = root(0).expect_non_asap().clone() + } = topk_limit(&root(0)).expect_non_asap().clone() else { - panic!("limit root") + panic!("limit under the root") }; assert!(!partition_by.keys().is_empty()); let NonASAPOp::Sort { @@ -496,12 +502,6 @@ mod tests { Payload::NonASAP(NonASAPOp::Limit { .. }) => "limit", _ => "other", }; - let sort_id = dag - .edges - .iter() - .find(|e| kind(e.producer) == "sort" && e.consumer == dag.roots[0]) - .expect("sort -> limit edge") - .producer; let producer_of = |consumer| { dag.edges .iter() @@ -509,6 +509,10 @@ mod tests { .unwrap() .producer }; + let limit_id = producer_of(dag.roots[0]); + assert_eq!(kind(limit_id), "limit"); + let sort_id = producer_of(limit_id); + assert_eq!(kind(sort_id), "sort"); assert_eq!(producer_of(sort_id), producer_of(dag.roots[1])); } diff --git a/crates/integration-tests/tests/physical_asap_optimization_sql.rs b/crates/integration-tests/tests/physical_asap_optimization_sql.rs index c593a1e9..020d87fb 100644 --- a/crates/integration-tests/tests/physical_asap_optimization_sql.rs +++ b/crates/integration-tests/tests/physical_asap_optimization_sql.rs @@ -20,10 +20,16 @@ fn approximate() -> AccuracyTarget { } } +fn names(node: &OperatorNode) -> Vec { + node.schema.fields.iter().map(|f| f.name.clone()).collect() +} + /// The `(sort column name, ascending)` of an exact top-k's `Sort`, if `root` /// is one (a heavy-hitter summary candidate has none). fn exact_topk_sort(root: &OperatorNode) -> Option<(String, bool)> { - let limit = root; + let NonASAPOp::Project { child: limit, .. } = root.non_asap()? else { + return None; + }; let NonASAPOp::Limit { n: Some(5), child: sort, @@ -46,9 +52,14 @@ fn exact_topk_sort(root: &OperatorNode) -> Option<(String, bool)> { /// Every Pass 1 choice of `sql` under `accuracy`, composed and physically /// optimized: each must succeed, every exact top-k must sort by `ranked` -/// descending. Returns how many +/// descending, and every root must output `root_names`. Returns how many /// candidates had an exact top-k. -async fn check_every_choice(sql: &str, accuracy: AccuracyTarget, ranked: &str) -> usize { +async fn check_every_choice( + sql: &str, + accuracy: AccuracyTarget, + ranked: &str, + root_names: &[&str], +) -> usize { let catalog = SqlCatalog::new().with_table( "metrics", Schema::new(vec![ @@ -107,6 +118,7 @@ async fn check_every_choice(sql: &str, accuracy: AccuracyTarget, ranked: &str) - let QueryRoot::Operator(physical) = &candidate.roots[0] else { panic!("operator root") }; + assert_eq!(names(physical), root_names, "{sql}, choice {choice:?}"); if let Some(sort) = exact_topk_sort(physical) { assert_eq!( sort, @@ -121,9 +133,9 @@ async fn check_every_choice(sql: &str, accuracy: AccuracyTarget, ranked: &str) - /// Each query under an exact and an approximate target; every exact top-k /// sorts by the ranked measure. -async fn check(sql: &str, ranked: &str) { +async fn check(sql: &str, ranked: &str, root_names: &[&str]) { for accuracy in [AccuracyTarget::Exact, approximate()] { - let exact = check_every_choice(sql, accuracy.clone(), ranked).await; + let exact = check_every_choice(sql, accuracy.clone(), ranked, root_names).await; assert!( exact > 0, "{sql} under {accuracy:?}: an exact top-k candidate" @@ -137,6 +149,7 @@ async fn group_key_named_value_ranks_by_count() { check( "SELECT value, COUNT(*) AS c FROM metrics GROUP BY value ORDER BY c DESC LIMIT 5", "count(Int64(1))", + &["topk_5"], ) .await; } @@ -147,6 +160,7 @@ async fn integer_group_key_ranks_by_count() { check( "SELECT port, COUNT(*) AS c FROM metrics GROUP BY port ORDER BY c DESC LIMIT 5", "count(Int64(1))", + &["topk_5"], ) .await; } @@ -158,6 +172,7 @@ async fn multi_key_ranks_by_count() { "SELECT host, port, COUNT(*) AS c FROM metrics GROUP BY host, port \ ORDER BY c DESC LIMIT 5", "count(Int64(1))", + &["topk_5"], ) .await; } @@ -168,6 +183,7 @@ async fn sum_ranking_ranks_by_sum() { check( "SELECT port, SUM(value) AS s FROM metrics GROUP BY port ORDER BY s DESC LIMIT 5", "sum(metrics.value)", + &["topk_5"], ) .await; } diff --git a/tools/dag-viewer/examples/planner-layering-example1.json b/tools/dag-viewer/examples/planner-layering-example1.json index 24bc139c..4273c743 100644 --- a/tools/dag-viewer/examples/planner-layering-example1.json +++ b/tools/dag-viewer/examples/planner-layering-example1.json @@ -3847,6 +3847,56 @@ "producer": 10, "role": "Input", "window": "NotApplicable" + }, + { + "consumer": 12, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 11, + "role": "Input", + "window": "NotApplicable" } ], "nodes": [ @@ -4793,11 +4843,79 @@ } } } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 12, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "topk_10", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Project": { + "child": 11, + "cols": [ + { + "alias": "job", + "expr": { + "Column": 2 + } + }, + { + "alias": "topk_10", + "expr": { + "Column": 1 + } + } + ], + "qualifier": null + } + } + } } ], "roots": [ 5, - 11 + 12 ], "scalar_roots": [] }, @@ -7906,6 +8024,10 @@ "cost": 0.001, "detail": "limit to 1000 rows" }, + "12": { + "cost": 0.001, + "detail": "operator: pass 1000 rows" + }, "2": { "cost": 4.0, "detail": "build exact Rate accumulator into 1000000 states: 4000000 rows x depth 1" @@ -7940,7 +8062,7 @@ } }, "source": "analytical-cost-v1 (illustrative statistics)", - "total": 79.40110000000001, + "total": 79.40210000000002, "unit": "cpu_ms_per_workload_evaluation" } }, From 434717a3af44d4b3cfc336b42a684691194a37da Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 13 Oct 2026 15:12:24 +0000 Subject: [PATCH 16/17] fix(planner): charge a NotMaterialized shared node once per consuming query Plan selection prices each node once per computation: a materialized node once, a NotMaterialized node once per query (or materialized node) that reaches it, not counting through a materialized ancestor. Per-node costs carry the multiplier. Also list the per-entity case among the exact top-k shapes Stage 2 cannot rewrite. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W7qG9aFyPij5uWsyAJCxDW --- crates/asap-aware-mapping/src/lib.rs | 5 +- .../src/physical_asap_optimization.rs | 5 +- .../asap-aware-mapping/src/plan_selection.rs | 149 ++++++++++++++++-- crates/devtools/src/bin/stage_pipeline.rs | 8 +- docs/user_guide_docs/stage-viewer.md | 5 +- .../examples/planner-layering-example1.json | 39 +++-- 6 files changed, 182 insertions(+), 29 deletions(-) diff --git a/crates/asap-aware-mapping/src/lib.rs b/crates/asap-aware-mapping/src/lib.rs index 44a0f645..734c124c 100644 --- a/crates/asap-aware-mapping/src/lib.rs +++ b/crates/asap-aware-mapping/src/lib.rs @@ -221,6 +221,7 @@ pub mod per_computation_candidates; /// producing the CandidatePhysicalASAPDAGs. pub mod physical_asap_optimization; -/// #509 Stage 3, plan selection (MVP): accuracy check, analytical pricing, -/// cheapest valid plan. +/// #509 Stage 3, plan selection (MVP): accuracy check, analytical pricing +/// (a `NotMaterialized` node once per query that reads it, a materialized +/// one once), cheapest valid plan. pub mod plan_selection; diff --git a/crates/asap-aware-mapping/src/physical_asap_optimization.rs b/crates/asap-aware-mapping/src/physical_asap_optimization.rs index 3a748dee..30dee2a3 100644 --- a/crates/asap-aware-mapping/src/physical_asap_optimization.rs +++ b/crates/asap-aware-mapping/src/physical_asap_optimization.rs @@ -19,8 +19,9 @@ //! `topk_k` carries the ranked value's type and nullability (the frontend //! declares a non-null `Utf8` placeholder); casting the value to `Utf8` would //! not make it the same answer. -//! An `Aggregate{[TopK]}` with other measures, a filter or `HAVING` has no -//! such rewrite, and its candidate fails. +//! An `Aggregate{[TopK]}` over a per-entity reduction, or with other +//! measures, a filter or `HAVING`, has no such rewrite, and its candidate +//! fails. //! A summary needs no rewrite: `SummaryAgg` → `SummaryEstimate` already is //! build → estimate. Every node keeps the guarantee Stage 1 composed for it. use std::collections::HashMap; diff --git a/crates/asap-aware-mapping/src/plan_selection.rs b/crates/asap-aware-mapping/src/plan_selection.rs index 6cce4db1..ef46cdff 100644 --- a/crates/asap-aware-mapping/src/plan_selection.rs +++ b/crates/asap-aware-mapping/src/plan_selection.rs @@ -9,9 +9,19 @@ //! over weights proven non-negative; a query without one constrains nothing. //! A miss rejects the candidate as invalid, with a reason; only this stage //! discards valid candidates. -//! Every valid candidate is priced node by node over its physical DAG, so a -//! node shared by several queries is charged once, and the cheapest is -//! selected. The rest are reported valid but costlier. +//! Every valid candidate is priced node by node over its physical DAG, and +//! the cheapest is selected. The rest are reported valid but costlier. +//! +//! A node is charged once per computation (#509): a materialized node +//! (`MatAtIngestion` or `MatAtQuery`) once, since its output is stored and +//! reused; a `NotMaterialized` node once per consumer that computes it, since +//! it is recomputed for each use and not stored. Its consumers are the +//! queries whose sub-DAG reaches it and the materialized nodes above it, +//! counted without looking through a materialized node: a node below one is +//! computed when that node is, so the materialized node counts as one +//! consumer. Stage 2 does not duplicate a shared node; the multiplier is +//! recorded on its cost ([`NodeCost::multiplier`]). Every node is +//! `NotMaterialized` until #604, so a shared node is charged once per query. //! //! Prices come from [`crate::analytical_cost::estimate_operator`] over edge //! statistics derived from the [`DataWorkload`] and a fixed default group @@ -21,7 +31,7 @@ //! A cost error on any candidate aborts selection; it is reachable only from //! bad declared rates. Latency bounds and deployment capabilities are not //! checked yet. -use std::collections::{BTreeMap, HashMap}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::rc::Rc; use asap_types::ir::physical_export::{ @@ -29,7 +39,8 @@ use asap_types::ir::physical_export::{ }; use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, QueryRoot, ScalarExpr}; use asap_types::post_asap::{ - FieldDataType, ResultGuarantee, SketchAlgorithm, SketchParams, SketchStatistic, WeightDomain, + FieldDataType, Materialization, ResultGuarantee, SketchAlgorithm, SketchParams, + SketchStatistic, WeightDomain, }; use asap_types::pre_asap::{DataType, Reduction, Schema}; use asap_types::types::AccuracyTarget; @@ -55,9 +66,12 @@ const DEFAULT_SERIES: u64 = 1_000; const DEFAULT_ROWS_PER_SECOND: f64 = 1_000.0; const DEFAULT_LOOKBACK_MS: u64 = 60_000; +/// `cost` is one computation's price times `multiplier`, the times the +/// node is computed per workload evaluation ([`computations`]). #[derive(Debug, Clone, PartialEq)] pub struct NodeCost { pub cost: f64, + pub multiplier: u64, pub detail: String, } @@ -269,8 +283,70 @@ struct Shape { rows_per_ms: f64, } -/// Price every node of `dag` once. Nodes are exported children first, so -/// each node's input statistics are known when it is reached. +/// How many times each node of `dag` is computed per workload evaluation: +/// 1 for a materialized node; for a `NotMaterialized` one, the number of +/// consumers that reach it, where a consumer is a query (operator or scalar +/// root) or a materialized node, and a walk stops at the first materialized +/// node it meets. A node read several times within one query's walk counts +/// once for it. +fn computations(dag: &PhysicalASAPDAG) -> HashMap { + let materialized: HashSet<_> = dag + .nodes + .iter() + .filter(|n| n.materialization != Materialization::NotMaterialized) + .map(|n| n.id) + .collect(); + let inputs = |id: PhysicalASAPNodeId| { + dag.edges + .iter() + .filter(move |e| e.consumer == id) + .map(|e| e.producer) + }; + // Each consumer's starting nodes: what a query reads, or a materialized + // node's inputs. + let consumers = dag + .roots + .iter() + .map(|root| vec![*root]) + .chain( + dag.scalar_roots + .iter() + .map(|(_, expr)| expr.operator_refs().into_iter().copied().collect()), + ) + .chain( + dag.nodes + .iter() + .filter(|n| materialized.contains(&n.id)) + .map(|n| inputs(n.id).collect()), + ); + let mut count: HashMap<_, u64> = HashMap::new(); + for start in consumers { + let mut seen = HashSet::new(); + let mut stack: Vec = start; + while let Some(id) = stack.pop() { + if materialized.contains(&id) || !seen.insert(id) { + continue; + } + *count.entry(id).or_default() += 1; + stack.extend(inputs(id)); + } + } + dag.nodes + .iter() + .map(|n| { + let times = if materialized.contains(&n.id) { + 1 + } else { + count.get(&n.id).copied().unwrap_or(0) + }; + (n.id, times) + }) + .collect() +} + +/// Price every node of `dag` once per computation ([`computations`]). Nodes +/// are exported children first, so each node's input statistics are known +/// when it is reached. fn price( dag: &PhysicalASAPDAG, data: &DataWorkload, @@ -295,6 +371,7 @@ fn price( version: "illustrative-v1".into(), }; let nodes: HashMap<_, _> = dag.nodes.iter().map(|n| (n.id, n)).collect(); + let multipliers = computations(dag); let mut output: HashMap = HashMap::new(); let mut per_node = BTreeMap::new(); for node in &dag.nodes { @@ -492,8 +569,16 @@ fn price( let cost = estimate .and_then(|estimate| estimate.calibrated_cost(&calibration)) .map_err(|error| (node.id, error))?; + let multiplier = multipliers[&node.id]; output.insert(node.id, out); - per_node.insert(node.id, NodeCost { cost, detail }); + per_node.insert( + node.id, + NodeCost { + cost: cost * multiplier as f64, + multiplier, + detail, + }, + ); } Ok(CandidateCost { total: per_node.values().map(|n| n.cost).sum(), @@ -688,7 +773,7 @@ mod tests { assert!(selection.rejected.iter().all(|r| r.valid)); } - /// Each node is priced once, under its DAG id, and the total is the sum; + /// Each node is priced under its DAG id, and the total is the sum; /// every candidate is either selected or rejected as costlier. The /// CountSketch+heap estimate has no analytical guarantee, so deployment /// evidence admits it. @@ -723,6 +808,52 @@ mod tests { assert_ne!(costlier[0].id, selection.selected); } + /// Query 1 is `sum(...)` over query 2's output. Every node reached by + /// both queries is `NotMaterialized`, so recomputed for each: charged + /// twice. Materializing query 2's root (`MatAtQuery`) charges it once, + /// and the nodes below it once, since they run only when it does. + #[test] + fn not_materialized_shared_nodes_are_charged_per_query() { + let outer = lower_promql("sum(sum_over_time(m[1m]))", AccuracyTarget::Exact); + let inner = outer.children()[0].clone(); + let mut candidate = optimize_physical_asap( + "L", + &[QueryRoot::Operator(outer), QueryRoot::Operator(inner)], + ) + .unwrap(); + let [outer, inner] = candidate.dag.roots[..] else { + panic!("two roots") + }; + let cost = price(&candidate.dag, &data()).unwrap(); + let multipliers: BTreeMap<_, _> = cost + .per_node + .iter() + .map(|(id, c)| (*id, c.multiplier)) + .collect(); + assert_eq!(multipliers[&outer], 1); + assert_eq!(multipliers[&inner], 2); + assert!(multipliers.len() > 2, "nodes below the shared root"); + for (id, times) in &multipliers { + if *id != outer { + assert_eq!(*times, 2, "node {id} is read by both queries"); + } + } + + let node = candidate + .dag + .nodes + .iter_mut() + .find(|n| n.id == inner) + .unwrap(); + node.materialization = Materialization::MatAtQuery(asap_types::ir::Medium::Memory); + let stored = price(&candidate.dag, &data()).unwrap(); + for (id, c) in &stored.per_node { + assert_eq!(c.multiplier, 1, "node {id} is computed once"); + assert_eq!(c.cost, cost.per_node[id].cost / multipliers[id] as f64); + } + assert!(stored.total < cost.total); + } + /// Every combination of Pass 1 candidates of one query, as Stage 2 /// candidates `P1`, `P2`, ... fn all_candidates(query: &str, target: &AccuracyTarget) -> Vec { diff --git a/crates/devtools/src/bin/stage_pipeline.rs b/crates/devtools/src/bin/stage_pipeline.rs index d843fa28..dab589cc 100644 --- a/crates/devtools/src/bin/stage_pipeline.rs +++ b/crates/devtools/src/bin/stage_pipeline.rs @@ -16,7 +16,11 @@ // cost; a composed logical candidate Stage 2 cannot implement is listed // under `pruned` with the reason; // - plan_selection: per-candidate costs, the selected candidate, and -// every other candidate as rejected (`valid: false`) or costlier. When +// every other candidate as rejected (`valid: false`) or costlier. Each +// node's `cost` is one computation's price times `multiplier`, the +// times it is computed: once if materialized, else once per query (or +// materialized node) that reads it, so a shared NotMaterialized node is +// charged once per query. When // selection fails, `selected` is null, with the rejections, or with // `error` for any other failure. // @@ -203,7 +207,7 @@ fn plan_selection_json(selection: &Result) -> Value { .map(|(node, c)| { ( node.to_string(), - json!({ "cost": c.cost, "detail": c.detail }), + json!({ "cost": c.cost, "multiplier": c.multiplier, "detail": c.detail }), ) }) .collect(); diff --git a/docs/user_guide_docs/stage-viewer.md b/docs/user_guide_docs/stage-viewer.md index 04b79612..b7faacf2 100644 --- a/docs/user_guide_docs/stage-viewer.md +++ b/docs/user_guide_docs/stage-viewer.md @@ -116,7 +116,10 @@ plans: ### Stage 3 · plans by cost -Every physical plan, cheapest first, with its total cost per second: +Every physical plan, cheapest first, with its total cost per second. +A node that is not materialized is recomputed for each query that reads it, +so a node shared by two queries is charged twice; a materialized node, at +ingestion or at query time, is charged once, and so are the nodes below it. - **✓ selected**: the plan ASAPPlanner chooses. - **valid · costlier**: meets every requirement, but costs more. diff --git a/tools/dag-viewer/examples/planner-layering-example1.json b/tools/dag-viewer/examples/planner-layering-example1.json index 4273c743..6218ff9c 100644 --- a/tools/dag-viewer/examples/planner-layering-example1.json +++ b/tools/dag-viewer/examples/planner-layering-example1.json @@ -8010,55 +8010,68 @@ "per_node": { "0": { "cost": 23.2, - "detail": "scan 4000000 samples" + "detail": "scan 4000000 samples", + "multiplier": 1 }, "1": { "cost": 4.0, - "detail": "time range 60s: pass 4000000 rows" + "detail": "time range 60s: pass 4000000 rows", + "multiplier": 1 }, "10": { "cost": 14.0, - "detail": "sort 1000000 rows in 100 partitions" + "detail": "sort 1000000 rows in 100 partitions", + "multiplier": 1 }, "11": { "cost": 0.001, - "detail": "limit to 1000 rows" + "detail": "limit to 1000 rows", + "multiplier": 1 }, "12": { "cost": 0.001, - "detail": "operator: pass 1000 rows" + "detail": "operator: pass 1000 rows", + "multiplier": 1 }, "2": { "cost": 4.0, - "detail": "build exact Rate accumulator into 1000000 states: 4000000 rows x depth 1" + "detail": "build exact Rate accumulator into 1000000 states: 4000000 rows x depth 1", + "multiplier": 1 }, "3": { "cost": 1.0, - "detail": "finalize 1000000 accumulators" + "detail": "finalize 1000000 accumulators", + "multiplier": 1 }, "4": { "cost": 1.0, - "detail": "build exact Sum accumulator into 100 states: 1000000 rows x depth 1" + "detail": "build exact Sum accumulator into 100 states: 1000000 rows x depth 1", + "multiplier": 1 }, "5": { "cost": 0.00009999999999999999, - "detail": "finalize 100 accumulators" + "detail": "finalize 100 accumulators", + "multiplier": 1 }, "6": { "cost": 23.2, - "detail": "scan 4000000 samples" + "detail": "scan 4000000 samples", + "multiplier": 1 }, "7": { "cost": 4.0, - "detail": "time range 60s: pass 4000000 rows" + "detail": "time range 60s: pass 4000000 rows", + "multiplier": 1 }, "8": { "cost": 4.0, - "detail": "build exact Sum accumulator into 1000000 states: 4000000 rows x depth 1" + "detail": "build exact Sum accumulator into 1000000 states: 4000000 rows x depth 1", + "multiplier": 1 }, "9": { "cost": 1.0, - "detail": "finalize 1000000 accumulators" + "detail": "finalize 1000000 accumulators", + "multiplier": 1 } }, "source": "analytical-cost-v1 (illustrative statistics)", From 1ad70f913b7479c36397aa7ab5605618197d3ca8 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 13 Oct 2026 15:23:52 +0000 Subject: [PATCH 17/17] docs(planner): nodes below a materialized node are charged once for it Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W7qG9aFyPij5uWsyAJCxDW --- docs/user_guide_docs/stage-viewer.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/user_guide_docs/stage-viewer.md b/docs/user_guide_docs/stage-viewer.md index b7faacf2..ff324c19 100644 --- a/docs/user_guide_docs/stage-viewer.md +++ b/docs/user_guide_docs/stage-viewer.md @@ -119,7 +119,8 @@ plans: Every physical plan, cheapest first, with its total cost per second. A node that is not materialized is recomputed for each query that reads it, so a node shared by two queries is charged twice; a materialized node, at -ingestion or at query time, is charged once, and so are the nodes below it. +ingestion or at query time, is charged once, and the nodes below it once for +it (plus once for each query that also reads them directly). - **✓ selected**: the plan ASAPPlanner chooses. - **valid · costlier**: meets every requirement, but costs more.