diff --git a/crates/asap-aware-mapping/src/accuracy/evidence.rs b/crates/asap-aware-mapping/src/accuracy/evidence.rs index f779d578..f770dd80 100644 --- a/crates/asap-aware-mapping/src/accuracy/evidence.rs +++ b/crates/asap-aware-mapping/src/accuracy/evidence.rs @@ -1,5 +1,6 @@ //! Scoped source contracts and evidence required by accuracy rules. use super::*; +use asap_types::ir::QueryRoot; /// A trusted source assertion scoped by `AccuracyEvidenceProvider` to one /// complete evaluation. Choosing this variant asserts the estimator and hash @@ -106,6 +107,17 @@ pub trait AccuracyEvidenceProvider { None } + /// The deployment's empirical accuracy for one candidate's whole query + /// output, as a guarantee. Plan selection (#509 Stage 3) accepts a + /// candidate whose output has no theoretical guarantee that meets the + /// target (e.g. a difference of two quantile estimates) only when this + /// shows the output meets it. Stage 1 composition never reads it: root + /// guarantees there come from the built-in theoretical model. `None` + /// means no evidence. + fn output_guarantee(&self, _output: &QueryRoot) -> Option { + None + } + fn propagation_stats( &self, _op: &CompositionOperator, diff --git a/crates/asap-aware-mapping/src/lib.rs b/crates/asap-aware-mapping/src/lib.rs index 41b58551..734c124c 100644 --- a/crates/asap-aware-mapping/src/lib.rs +++ b/crates/asap-aware-mapping/src/lib.rs @@ -215,3 +215,13 @@ pub mod maintained_population; /// Stage 1, Pass 1: per-computation candidate generation. No execution timing is /// assigned: that is a Stage 2 materialization decision. pub mod per_computation_candidates; + +/// #509 Stage 2, physical ASAP-aware optimization (MVP): materialization +/// (every sub-DAG `NotMaterialized`) and physical operator implementation, +/// producing the CandidatePhysicalASAPDAGs. +pub mod physical_asap_optimization; + +/// #509 Stage 3, plan selection (MVP): accuracy check, analytical pricing +/// (a `NotMaterialized` node once per query that reads it, a materialized +/// one once), cheapest valid plan. +pub mod plan_selection; diff --git a/crates/asap-aware-mapping/src/physical_asap_optimization.rs b/crates/asap-aware-mapping/src/physical_asap_optimization.rs new file mode 100644 index 00000000..30dee2a3 --- /dev/null +++ b/crates/asap-aware-mapping/src/physical_asap_optimization.rs @@ -0,0 +1,559 @@ +//! #509 Stage 2, physical ASAP-aware optimization (MVP): turns one +//! CandidateLogicalASAPDAG into CandidatePhysicalASAPDAGs by materialization +//! and physical operator implementation. Materialization offers one option +//! for now: every sub-DAG is `Materialization::NotMaterialized` and runs at +//! query time, so each logical candidate yields exactly one physical +//! candidate. Enumerating +//! `MatAtIngestion(medium)` and `MatAtQuery(medium)` per sub-DAG +//! (Algorithm 3) comes in #604. A candidate this stage cannot implement +//! fails with a [`PhysicalASAPOptimizationError`], and the caller prunes that candidate alone, +//! with the error as its reason. +//! +//! The runtime has one implementation per logical operator except exact +//! top-k, which it cannot run as `Aggregate{[TopK]}`. Stage 2 records that +//! choice in the DAG by rewriting it to a per-group sort followed by a +//! per-group limit, the shape the PromQL frontend uses for generic `topk`. +//! A `Project` over the limited rows restores the logical `[keys.., topk_k]` +//! columns, so an exact root has the same column names as a summary +//! candidate's and as the frontend declares, and no input column leaks. +//! `topk_k` carries the ranked value's type and nullability (the frontend +//! declares a non-null `Utf8` placeholder); casting the value to `Utf8` would +//! not make it the same answer. +//! An `Aggregate{[TopK]}` over a per-entity reduction, or with other +//! measures, a filter or `HAVING`, has no such rewrite, and its candidate +//! fails. +//! A summary needs no rewrite: `SummaryAgg` → `SummaryEstimate` already is +//! build → estimate. Every node keeps the guarantee Stage 1 composed for it. +use std::collections::HashMap; +use std::rc::Rc; + +use asap_types::ir::physical_export::{ + compile_physical_asap_workload_with_node_ids, PhysicalASAPDAG, +}; +use asap_types::ir::{ + apply_materialization_timings, ASAPOp, MaterializationAssignment, NonASAPOp, Operator, + OperatorNode, ProjectItem, QueryRoot, ScalarExpr, SchemaDerivationError, SortKey, TimingMemo, +}; +use asap_types::post_asap::{ExecutionDataStateError, Materialization}; +use asap_types::pre_asap::column_resolution::resolve_column_ref; +use asap_types::pre_asap::expr_ir::ColumnRef; +use asap_types::pre_asap::schema::ColumnId; +use asap_types::pre_asap::{AggIntent, Reduction}; +use thiserror::Error; + +/// One CandidatePhysicalASAPDAG (Stage 2 output), derived from exactly one +/// CandidateLogicalASAPDAG (Stage 1 output). +/// `roots` are the timed query roots (one per query, in workload order) +/// that `dag` exports. +#[derive(Debug, Clone)] +pub struct PhysicalCandidate { + pub id: String, + pub from_logical: String, + pub label: String, + pub roots: Vec, + pub dag: PhysicalASAPDAG, +} + +#[derive(Debug, Error)] +pub enum PhysicalASAPOptimizationError { + #[error(transparent)] + Structure(#[from] SchemaDerivationError), + #[error(transparent)] + Timing(#[from] ExecutionDataStateError), + #[error("exact top-k has no Sort + Limit implementation: {0}")] + ExactTopK(String), +} + +/// Stage 2 for one CandidateLogicalASAPDAG: implement every operator +/// (physical operator implementation) and time it with every +/// sub-DAG `NotMaterialized`. `id` and `label` are left empty for the caller +/// to name. Sharing between `roots` is preserved: one memo serves the whole +/// workload. +pub fn optimize_physical_asap( + from_logical: &str, + roots: &[QueryRoot], +) -> Result { + let mut memo = HashMap::new(); + let implemented = map_roots(roots, &mut |node| implement(node, &mut memo))?; + let assignment = MaterializationAssignment::with_default(Materialization::NotMaterialized); + let mut timing = TimingMemo::new(); + let timed = map_roots(&implemented, &mut |node| { + Ok(apply_materialization_timings( + node, + &assignment, + &mut timing, + )?) + })?; + let dag = compile_physical_asap_workload_with_node_ids(&timed)?.dag; + Ok(PhysicalCandidate { + id: String::new(), + from_logical: from_logical.to_string(), + label: String::new(), + roots: timed, + dag, + }) +} + +/// `roots` with every operator node they read replaced by `f`'s result. +fn map_roots( + roots: &[QueryRoot], + f: &mut impl FnMut(&Rc) -> Result, PhysicalASAPOptimizationError>, +) -> Result, PhysicalASAPOptimizationError> { + roots + .iter() + .map(|root| match root { + QueryRoot::Operator(node) => Ok(QueryRoot::Operator(f(node)?)), + QueryRoot::Scalar(expr) => { + let mut mapped = HashMap::new(); + for node in expr.operator_refs() { + mapped.insert(Rc::as_ptr(node), f(node)?); + } + Ok(QueryRoot::Scalar(expr.map_operator_refs(&mut |node| { + mapped[&Rc::as_ptr(node)].clone() + }))) + } + }) + .collect() +} + +/// Each node's implementation. +type Memo = HashMap<*const OperatorNode, Rc>; + +fn implement( + node: &Rc, + memo: &mut Memo, +) -> Result, PhysicalASAPOptimizationError> { + if let Some(done) = memo.get(&Rc::as_ptr(node)) { + return Ok(done.clone()); + } + for child in node.children() { + implement(child, memo)?; + } + let implemented = match node.non_asap() { + Some(NonASAPOp::Aggregate { measures, .. }) + if measures.iter().any(|m| matches!(m, AggIntent::TopK { .. })) => + { + implement_topk(node, memo)? + } + _ => rebuilt(node, memo)?, + }; + memo.insert(Rc::as_ptr(node), implemented.clone()); + Ok(implemented) +} + +/// Exact top-k as `Limit(Sort)` per group, which outputs the ranked rows of +/// the child, under a `Project` restoring the logical `[keys.., topk_k]` +/// output (the ranked value under `topk_k`): a root returns those columns, +/// and a node reading the top-k resolved its columns against them. +fn implement_topk( + node: &Rc, + memo: &Memo, +) -> Result, PhysicalASAPOptimizationError> { + let unsupported = |why: &str| PhysicalASAPOptimizationError::ExactTopK(why.to_string()); + let Some(NonASAPOp::Aggregate { + reduction, + measures, + filters, + having, + child, + .. + }) = node.non_asap() + else { + unreachable!("implement_topk is called on an Aggregate") + }; + let Reduction::Reduce(keys) = reduction else { + return Err(unsupported("per-entity reduction")); + }; + let [AggIntent::TopK { k, .. }] = measures.as_slice() else { + return Err(unsupported("measures besides the top-k")); + }; + if filters.iter().any(Option::is_some) { + return Err(unsupported("filtered measure")); + } + if having.is_some() { + return Err(unsupported("HAVING")); + } + let child = memo[&Rc::as_ptr(child)].clone(); + let value = ranked_column(&child)?; + let sorted = Rc::new( + OperatorNode::new(Operator::NonASAP(NonASAPOp::Sort { + keys: vec![SortKey { + expr: ScalarExpr::Column(value), + ascending: false, + nulls_first: false, + }], + partition_by: keys.clone(), + child: child.clone(), + }))? + .with_guarantee(child.guarantee.clone()), + ); + let limited = Rc::new( + OperatorNode::new(Operator::NonASAP(NonASAPOp::Limit { + n: Some(*k), + offset: 0, + partition_by: keys.clone(), + child: sorted, + }))? + .with_guarantee(node.guarantee.clone()), + ); + let (ranked, key_fields) = node + .schema + .fields + .split_last() + .expect("an aggregate outputs its measure"); + let mut cols = key_fields + .iter() + .map(|field| { + let column = resolve_column_ref(&ColumnRef::Named(field.name.clone()), &child.schema) + .map_err(|e| unsupported(&format!("group key {}: {e}", field.name)))?; + Ok(ProjectItem { + alias: Some(field.name.clone()), + expr: ScalarExpr::Column(column), + }) + }) + .collect::, PhysicalASAPOptimizationError>>()?; + cols.push(ProjectItem { + alias: Some(ranked.name.clone()), + expr: ScalarExpr::Column(value), + }); + Ok(Rc::new( + OperatorNode::new(Operator::NonASAP(NonASAPOp::Project { + cols, + qualifier: None, + child: limited, + }))? + .with_guarantee(node.guarantee.clone()), + )) +} + +/// The column exact top-k ranks `child` by. Over a one-measure grouped +/// aggregate (the SQL heavy-hitter shape and PromQL `topk(k, sum by ..)`) it +/// is that measure, the last column after the group keys, whichever Stage 1 +/// candidate realizes it: the aggregate itself, or a `SummaryAgg` read by +/// `FinalizeExactAccumulator` or `SummaryEstimate`, under the `Project` +/// restoring its names (a renaming) and a value `HAVING`'s `Filter`. Over per-series rows +/// (PromQL `topk by (..) (k, rows)`) it is the sample value. +fn ranked_column(child: &OperatorNode) -> Result { + let mut node = child; + let reduction = loop { + match &node.operator { + // Only a renaming keeps the measure last. + Operator::NonASAP(NonASAPOp::Project { cols, child, .. }) + if cols.len() == child.schema.fields.len() + && cols + .iter() + .enumerate() + .all(|(i, item)| item.expr == ScalarExpr::Column(i)) => + { + node = child + } + Operator::NonASAP(NonASAPOp::Filter { child, .. }) + | Operator::ASAP( + ASAPOp::FinalizeExactAccumulator { child } + | ASAPOp::SummaryEstimate { + summary_input: child, + .. + }, + ) => node = child, + Operator::ASAP(ASAPOp::SummaryAgg { reduction, .. }) => break Some(reduction), + Operator::NonASAP(NonASAPOp::Aggregate { + reduction, + measures, + .. + }) if measures.len() == 1 => break Some(reduction), + _ => break None, + } + }; + match reduction { + Some(Reduction::Reduce(_)) => Ok(child.schema.fields.len() - 1), + _ => resolve_column_ref(&ColumnRef::SampleValue, &child.schema).map_err(|e| { + PhysicalASAPOptimizationError::ExactTopK(format!("no ranked column: {e}")) + }), + } +} + +/// `node` over its implemented children; the same `Rc` when none changed. +fn rebuilt( + node: &Rc, + memo: &Memo, +) -> Result, PhysicalASAPOptimizationError> { + let input = |child: &Rc| memo[&Rc::as_ptr(child)].clone(); + let changed = node + .children() + .iter() + .any(|child| !Rc::ptr_eq(child, &input(child))); + Ok(if changed { + Rc::new( + node.with_new_children(|child| input(child))? + .with_guarantee(node.guarantee.clone()), + ) + } else { + node.clone() + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_support::lower_promql; + use asap_types::ir::physical_export::PhysicalASAPNodeId; + use asap_types::ir::physical_export::PhysicalASAPOperatorPayload as Payload; + use asap_types::ir::ASAPOp; + use asap_types::post_asap::ExecutionTiming; + use asap_types::types::AccuracyTarget; + + fn operator_root(candidate: &PhysicalCandidate, i: usize) -> Rc { + match &candidate.roots[i] { + QueryRoot::Operator(node) => node.clone(), + QueryRoot::Scalar(_) => panic!("operator root"), + } + } + + fn names(node: &OperatorNode) -> Vec { + node.schema.fields.iter().map(|f| f.name.clone()).collect() + } + + /// The `Limit` under an exact top-k's restoring `Project`. + fn topk_limit(project: &OperatorNode) -> Rc { + let NonASAPOp::Project { child, .. } = project.expect_non_asap() else { + panic!("project, found {}", project.operator.kind_name()) + }; + child.clone() + } + + /// The single sort key of the `Sort` under the `Limit` at `limit`. + fn topk_sort_key(limit: &OperatorNode) -> (ScalarExpr, bool) { + let NonASAPOp::Limit { child: sort, .. } = limit.expect_non_asap() else { + panic!("limit, found {}", limit.operator.kind_name()) + }; + let NonASAPOp::Sort { keys, .. } = sort.expect_non_asap() else { + panic!("sort below limit") + }; + let [key] = keys.as_slice() else { + panic!("one sort key") + }; + (key.expr.clone(), key.ascending) + } + + /// Example 1's shape, `topk by (job) (k, per-series rows)`, sorts each + /// group by the sample value, descending. + #[test] + fn promql_topk_over_series_sorts_by_value_descending() { + let root = lower_promql( + "topk by (job) (10, sum_over_time(m[1m]))", + AccuracyTarget::Exact, + ); + let candidate = optimize_physical_asap("L1", &[root.clone().into()]).unwrap(); + let physical = operator_root(&candidate, 0); + assert_eq!(names(&physical), names(&root)); + let limit = topk_limit(&physical); + assert_eq!(limit.schema.fields[1].name, "value"); + assert_eq!(topk_sort_key(&limit), (ScalarExpr::Column(1), false)); + } + + /// `topk(k, sum by (job) (..))` sorts by the summed measure. + #[test] + fn promql_topk_over_grouped_aggregate_sorts_by_its_measure() { + let root = lower_promql("topk(5, sum by (job) (m))", AccuracyTarget::Exact); + let candidate = optimize_physical_asap("L1", &[root.into()]).unwrap(); + let physical = operator_root(&candidate, 0); + assert_eq!(names(&physical), ["topk_5"]); + let limit = topk_limit(&physical); + assert_eq!(names(&limit), ["job", "sum"]); + assert_eq!(topk_sort_key(&limit), (ScalarExpr::Column(1), false)); + } + + /// A node reading an exact top-k keeps resolving the columns it bound + /// against the logical `[keys.., topk_k]` output: `sum by (job)` still + /// groups by `job`, not by whatever column now sits at its index. + #[test] + fn consumer_of_exact_topk_reads_its_logical_columns() { + let root = lower_promql( + "sum by (job) (topk by (job) (10, sum_over_time(m[1m])))", + AccuracyTarget::Exact, + ); + let candidate = optimize_physical_asap("L1", &[root.clone().into()]).unwrap(); + let physical = operator_root(&candidate, 0); + assert_eq!(names(&physical), names(&root)); + let NonASAPOp::Aggregate { child: input, .. } = physical.expect_non_asap() else { + panic!("sum root") + }; + let NonASAPOp::Project { + cols, child: limit, .. + } = input.expect_non_asap() + else { + panic!("project over the top-k rows") + }; + let read: Vec<_> = cols + .iter() + .map(|item| { + let ScalarExpr::Column(c) = item.expr else { + panic!("column") + }; + ( + item.alias.clone().unwrap(), + limit.schema.fields[c].name.clone(), + ) + }) + .collect(); + assert_eq!( + read, + [ + ("job".to_string(), "job".to_string()), + ("topk_10".to_string(), "value".to_string()) + ] + ); + assert_eq!(topk_sort_key(limit), (ScalarExpr::Column(1), false)); + } + + /// An `Aggregate{[TopK]}` that Sort + Limit cannot express (another + /// measure, `HAVING`) fails, so its candidate is pruned with the reason. + #[test] + fn topk_with_extra_measure_or_having_fails() { + use crate::test_support::{aggregate, metric_scan}; + use asap_types::ir::Predicate; + use asap_types::pre_asap::expr_ir::ScalarValue; + let topk = AggIntent::TopK { + k: 3, + accuracy: AccuracyTarget::Exact, + }; + let count = AggIntent::Count { + accuracy: AccuracyTarget::Exact, + }; + let by_job = || Reduction::by(vec![2]); + let cases = [ + aggregate( + by_job(), + vec![topk.clone(), count], + vec![], + None, + metric_scan(&["job"]), + ), + aggregate( + by_job(), + vec![topk], + vec![], + Some(Predicate(ScalarExpr::Literal(ScalarValue::Boolean(true)))), + metric_scan(&["job"]), + ), + ]; + for root in cases { + let error = optimize_physical_asap("L1", &[root.into()]).unwrap_err(); + assert!( + matches!(error, PhysicalASAPOptimizationError::ExactTopK(_)), + "{error}" + ); + } + } + + /// Exact `topk by (job)` becomes Limit(Sort) partitioned by `job`, and a + /// child shared by two roots stays one node (one `Rc`, one DAG node). + #[test] + fn exact_topk_becomes_sort_then_limit_and_keeps_shared_child() { + let exact = AccuracyTarget::Exact; + let topk = lower_promql("topk by (job) (10, sum_over_time(m[1m]))", exact.clone()); + let sum = lower_promql("sum by (job) (sum_over_time(m[1m]))", exact); + // Share Q2's per-series child with Q1, as Stage 1 sharing would. + let NonASAPOp::Aggregate { child: shared, .. } = topk.expect_non_asap() else { + panic!("topk aggregate") + }; + let sum = Rc::new(sum.with_new_children(|_| shared.clone()).unwrap()); + let candidate = optimize_physical_asap("L1", &[topk.into(), sum.into()]).unwrap(); + let root = |i: usize| match &candidate.roots[i] { + QueryRoot::Operator(node) => node.clone(), + QueryRoot::Scalar(_) => panic!("operator root"), + }; + + let NonASAPOp::Limit { + n: Some(10), + partition_by, + child: sort, + .. + } = topk_limit(&root(0)).expect_non_asap().clone() + else { + panic!("limit under the root") + }; + assert!(!partition_by.keys().is_empty()); + let NonASAPOp::Sort { + child: below_sort, + partition_by: sort_partition, + .. + } = sort.expect_non_asap() + else { + panic!("sort below limit") + }; + assert_eq!(*sort_partition, partition_by); + let NonASAPOp::Aggregate { + child: below_sum, .. + } = root(1).expect_non_asap().clone() + else { + panic!("sum root") + }; + assert!(Rc::ptr_eq(below_sort, &below_sum)); + + let dag = &candidate.dag; + let kind = |id: PhysicalASAPNodeId| match &dag + .nodes + .iter() + .find(|n| n.id == id) + .unwrap() + .payload + { + Payload::NonASAP(NonASAPOp::Sort { .. }) => "sort", + Payload::NonASAP(NonASAPOp::Limit { .. }) => "limit", + _ => "other", + }; + let producer_of = |consumer| { + dag.edges + .iter() + .find(|e| e.consumer == consumer) + .unwrap() + .producer + }; + let limit_id = producer_of(dag.roots[0]); + assert_eq!(kind(limit_id), "limit"); + let sort_id = producer_of(limit_id); + assert_eq!(kind(sort_id), "sort"); + assert_eq!(producer_of(sort_id), producer_of(dag.roots[1])); + } + + /// With summaries chosen, every node, the summary build included, runs at + /// query time. + #[test] + fn everything_runs_at_query_time() { + let root = lower_promql( + "topk by (job) (10, sum_over_time(m[1m]))", + AccuracyTarget::EpsilonDelta { + epsilon: 0.01, + delta: 0.001, + }, + ); + let inventory = crate::per_computation_candidates::generate_per_computation_candidates( + vec![(0, QueryRoot::Operator(root))], + ) + .unwrap(); + // A summary (the last candidate) for every target. + let choice: Vec<_> = inventory + .targets + .iter() + .map(|t| t.choices.candidates.len() - 1) + .collect(); + let roots: Vec<_> = + crate::per_computation_candidates::compose_logical_candidate(&inventory, &choice) + .unwrap() + .into_iter() + .map(|(_, root)| root) + .collect(); + let candidate = optimize_physical_asap("L1", &roots).unwrap(); + assert!(candidate + .dag + .nodes + .iter() + .any(|n| matches!(n.payload, Payload::ASAP(ASAPOp::SummaryAgg { .. })))); + assert!(candidate + .dag + .nodes + .iter() + .all(|n| n.output_state.timing == ExecutionTiming::QueryTime)); + } +} diff --git a/crates/asap-aware-mapping/src/plan_selection.rs b/crates/asap-aware-mapping/src/plan_selection.rs new file mode 100644 index 00000000..ef46cdff --- /dev/null +++ b/crates/asap-aware-mapping/src/plan_selection.rs @@ -0,0 +1,997 @@ +//! #509 Stage 3, plan selection (MVP): the only stage that computes cost. +//! +//! Each CandidatePhysicalASAPDAG from Stage 2 is checked against every +//! query's accuracy target: the query's output meets it when its root +//! guarantee does (theoretical: Stage 1 composes it with the built-in +//! accuracy model), or when the deployment's empirical evidence shows this +//! candidate's output does (`AccuracyEvidenceProvider::output_guarantee`, +//! read only here). For a query with a target, Count-Min is admitted only +//! over weights proven non-negative; a query without one constrains nothing. +//! A miss rejects the candidate as invalid, with a reason; only this stage +//! discards valid candidates. +//! Every valid candidate is priced node by node over its physical DAG, and +//! the cheapest is selected. The rest are reported valid but costlier. +//! +//! A node is charged once per computation (#509): a materialized node +//! (`MatAtIngestion` or `MatAtQuery`) once, since its output is stored and +//! reused; a `NotMaterialized` node once per consumer that computes it, since +//! it is recomputed for each use and not stored. Its consumers are the +//! queries whose sub-DAG reaches it and the materialized nodes above it, +//! counted without looking through a materialized node: a node below one is +//! computed when that node is, so the materialized node counts as one +//! consumer. Stage 2 does not duplicate a shared node; the multiplier is +//! recorded on its cost ([`NodeCost::multiplier`]). Every node is +//! `NotMaterialized` until #604, so a shared node is charged once per query. +//! +//! Prices come from [`crate::analytical_cost::estimate_operator`] over edge +//! statistics derived from the [`DataWorkload`] and a fixed default group +//! count; summary build and estimation are priced as rows × sketch depth and +//! groups × k. These numbers are illustrative, not calibrated: a binary, +//! join or concat node is priced from its first input only (MVP; follow-up). +//! A cost error on any candidate aborts selection; it is reachable only from +//! bad declared rates. Latency bounds and deployment capabilities are not +//! checked yet. +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::rc::Rc; + +use asap_types::ir::physical_export::{ + PhysicalASAPDAG, PhysicalASAPNodeId, PhysicalASAPOperatorPayload as Payload, +}; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, QueryRoot, ScalarExpr}; +use asap_types::post_asap::{ + FieldDataType, Materialization, ResultGuarantee, SketchAlgorithm, SketchParams, + SketchStatistic, WeightDomain, +}; +use asap_types::pre_asap::{DataType, Reduction, Schema}; +use asap_types::types::AccuracyTarget; +use asap_types::workload::DataWorkload; +use thiserror::Error; + +use crate::analytical_cost::{ + estimate_operator, AnalyticalCostError, PhysicalOperator, ResourceCalibration, ResourceEstimate, +}; +use crate::physical_asap_optimization::PhysicalCandidate; +use crate::physical_operator_statistics::{ + EdgeStatistics, OperatorStatistics, PartitionStatistics, UnaryEdgeStatistics, +}; +use crate::PlanningModels; + +pub const COST_UNIT: &str = "cpu_ms_per_workload_evaluation"; +pub const COST_SOURCE: &str = "analytical-cost-v1 (illustrative statistics)"; + +/// Groups assumed for every `by (...)` reduction, absent group-count evidence. +const DEFAULT_GROUP_COUNT: u64 = 100; +/// Used only when the data workload does not declare them. +const DEFAULT_SERIES: u64 = 1_000; +const DEFAULT_ROWS_PER_SECOND: f64 = 1_000.0; +const DEFAULT_LOOKBACK_MS: u64 = 60_000; + +/// `cost` is one computation's price times `multiplier`, the times the +/// node is computed per workload evaluation ([`computations`]). +#[derive(Debug, Clone, PartialEq)] +pub struct NodeCost { + pub cost: f64, + pub multiplier: u64, + pub detail: String, +} + +/// Whole-workload cost of one candidate: one entry per DAG node, `total` is +/// their sum. +#[derive(Debug, Clone, PartialEq)] +pub struct CandidateCost { + pub total: f64, + pub unit: &'static str, + pub source: &'static str, + pub per_node: BTreeMap, +} + +/// A candidate that was not selected. `valid == false`: it failed a check; +/// `true`: it lost on cost. +#[derive(Debug, Clone, PartialEq)] +pub struct Rejection { + pub id: String, + pub valid: bool, + pub reason: String, +} + +/// Costs are present for valid candidates only. +#[derive(Debug, Clone, PartialEq)] +pub struct Selection { + pub selected: String, + pub costs: BTreeMap, + pub rejected: Vec, +} + +#[derive(Debug, Error)] +pub enum SelectionError { + #[error("candidate {candidate} has {roots} roots but {targets} accuracy targets were given")] + TargetCount { + candidate: String, + roots: usize, + targets: usize, + }, + #[error("candidate {candidate}, node {node:?}: {error}")] + Cost { + candidate: String, + node: PhysicalASAPNodeId, + error: AnalyticalCostError, + }, + #[error("no valid candidate: {0:?}")] + NoValidCandidate(Vec), +} + +/// Stage 3 (plan selection): reject candidates that miss a query's accuracy +/// target, price the rest and +/// select the cheapest (the first on ties). `targets[i]` is the requirement +/// of `candidate.roots[i]`; `None` imposes none. +pub fn select_plan( + cands: &[PhysicalCandidate], + targets: &[Option], + data: &DataWorkload, + models: PlanningModels<'_>, +) -> Result { + let mut costs = BTreeMap::new(); + let mut rejected = Vec::new(); + let mut best: Option<(&str, f64)> = None; + for candidate in cands { + if candidate.roots.len() != targets.len() { + return Err(SelectionError::TargetCount { + candidate: candidate.id.clone(), + roots: candidate.roots.len(), + targets: targets.len(), + }); + } + if let Some(reason) = accuracy_violation(candidate, targets, &models) { + rejected.push(Rejection { + id: candidate.id.clone(), + valid: false, + reason, + }); + continue; + } + let cost = price(&candidate.dag, data).map_err(|(node, error)| SelectionError::Cost { + candidate: candidate.id.clone(), + node, + error, + })?; + if best.is_none_or(|(_, total)| cost.total < total) { + best = Some((&candidate.id, cost.total)); + } + costs.insert(candidate.id.clone(), cost); + } + let Some((selected, best_total)) = best else { + return Err(SelectionError::NoValidCandidate(rejected)); + }; + for candidate in cands { + if candidate.id != selected && costs.contains_key(&candidate.id) { + rejected.push(Rejection { + id: candidate.id.clone(), + valid: true, + reason: format!( + "costlier: {:.3} vs {:.3} {COST_UNIT}", + costs[&candidate.id].total, best_total + ), + }); + } + } + Ok(Selection { + selected: selected.to_string(), + costs, + rejected, + }) +} + +/// The first query whose output is not shown to meet its target, as a +/// reason. An output meets its target when its theoretical root guarantee +/// (composed in Stage 1 with the built-in model) satisfies it, or when the +/// deployment's empirical evidence shows that this candidate's output does +/// ([`crate::accuracy::AccuracyEvidenceProvider::output_guarantee`]). A +/// certified estimate inside the plan is not enough on its own: a difference +/// or ratio of certified estimates can be unbounded. Count-Min is admitted +/// only over update weights proven non-negative, which its bound assumes. +fn accuracy_violation( + candidate: &PhysicalCandidate, + targets: &[Option], + models: &PlanningModels<'_>, +) -> Option { + for (query, (root, target)) in candidate.roots.iter().zip(targets).enumerate() { + let query = query + 1; + let Some(target) = target else { continue }; + for node in root_operators(root) + .iter() + .flat_map(|node| OperatorNode::reachable(node)) + { + let Operator::ASAP(ASAPOp::SummaryAgg { family, input, .. }) = &node.operator else { + continue; + }; + if matches!(family, FieldDataType::Sketch(kind, _) + if matches!(kind.algorithm(), SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap)) + && !matches!(input.weight_domain, WeightDomain::NonNegative { .. }) + { + return Some(format!( + "q{query}: {} needs non-negative update weights, and these are not proven \ + non-negative", + family_name(family) + )); + } + } + let guarantee = root_guarantee(root); + let meets = |g: &ResultGuarantee| models.accuracy.satisfies(g, target); + if guarantee.as_ref().is_some_and(meets) + || models + .evidence + .output_guarantee(root) + .as_ref() + .is_some_and(meets) + { + continue; + } + return Some(match guarantee { + Some(g) => format!( + "q{query}: the output guarantees bound {:?}, failure probability {:?}, which \ + misses target {target:?}, and the deployment has no evidence that it meets it", + g.bound.evaluate(), + g.failure_probability.evaluate(), + ), + None => format!( + "q{query}: the output has no guarantee, and the deployment has no evidence \ + that it meets target {target:?}" + ), + }); + } + None +} + +/// The operator nodes a query root reads. +fn root_operators(root: &QueryRoot) -> Vec<&Rc> { + match root { + QueryRoot::Operator(node) => vec![node], + QueryRoot::Scalar(expr) => expr.operator_refs(), + } +} + +/// The theoretical guarantee of a query's output: the root node's own; +/// `scalar(v)` alone inherits `v`'s; any other scalar root is exact when +/// every operator it reads is, and has none otherwise (#669). +fn root_guarantee(root: &QueryRoot) -> Option { + match root { + QueryRoot::Operator(node) => node.guarantee.clone(), + QueryRoot::Scalar(ScalarExpr::PromqlScalarFromVector(node)) => node.guarantee.clone(), + QueryRoot::Scalar(expr) => expr + .operator_refs() + .into_iter() + .all(|node| { + node.guarantee + .as_ref() + .is_some_and(ResultGuarantee::is_exact) + }) + .then(|| ResultGuarantee::exact("scalar over exact inputs")), + } +} + +fn family_name(family: &FieldDataType) -> String { + match family { + FieldDataType::Sketch(kind, _) => format!("{:?}", kind.algorithm()), + FieldDataType::ExactAggregate(kind, _) => format!("exact {kind:?} accumulator"), + other => format!("{other:?}"), + } +} + +/// Statistics the analytical model needs, derived once per workload. +struct Shape { + series: u64, + rows_per_ms: f64, +} + +/// How many times each node of `dag` is computed per workload evaluation: +/// 1 for a materialized node; for a `NotMaterialized` one, the number of +/// consumers that reach it, where a consumer is a query (operator or scalar +/// root) or a materialized node, and a walk stops at the first materialized +/// node it meets. A node read several times within one query's walk counts +/// once for it. +fn computations(dag: &PhysicalASAPDAG) -> HashMap { + let materialized: HashSet<_> = dag + .nodes + .iter() + .filter(|n| n.materialization != Materialization::NotMaterialized) + .map(|n| n.id) + .collect(); + let inputs = |id: PhysicalASAPNodeId| { + dag.edges + .iter() + .filter(move |e| e.consumer == id) + .map(|e| e.producer) + }; + // Each consumer's starting nodes: what a query reads, or a materialized + // node's inputs. + let consumers = dag + .roots + .iter() + .map(|root| vec![*root]) + .chain( + dag.scalar_roots + .iter() + .map(|(_, expr)| expr.operator_refs().into_iter().copied().collect()), + ) + .chain( + dag.nodes + .iter() + .filter(|n| materialized.contains(&n.id)) + .map(|n| inputs(n.id).collect()), + ); + let mut count: HashMap<_, u64> = HashMap::new(); + for start in consumers { + let mut seen = HashSet::new(); + let mut stack: Vec = start; + while let Some(id) = stack.pop() { + if materialized.contains(&id) || !seen.insert(id) { + continue; + } + *count.entry(id).or_default() += 1; + stack.extend(inputs(id)); + } + } + dag.nodes + .iter() + .map(|n| { + let times = if materialized.contains(&n.id) { + 1 + } else { + count.get(&n.id).copied().unwrap_or(0) + }; + (n.id, times) + }) + .collect() +} + +/// Price every node of `dag` once per computation ([`computations`]). Nodes +/// are exported children first, so each node's input statistics are known +/// when it is reached. +fn price( + dag: &PhysicalASAPDAG, + data: &DataWorkload, +) -> Result { + let series = data + .input_cardinality + .value + .unwrap_or(DEFAULT_SERIES) + .max(1); + let shape = Shape { + series, + rows_per_ms: data + .ingestion_rate + .value + .map_or(DEFAULT_ROWS_PER_SECOND, |rate| rate.0) + / 1_000.0, + }; + let calibration = ResourceCalibration { + cost_per_cpu_op: 1e-6, + cost_per_scan_byte: 1e-7, + cost_per_retained_byte: 0.0, + version: "illustrative-v1".into(), + }; + let nodes: HashMap<_, _> = dag.nodes.iter().map(|n| (n.id, n)).collect(); + let multipliers = computations(dag); + let mut output: HashMap = HashMap::new(); + let mut per_node = BTreeMap::new(); + for node in &dag.nodes { + let inputs: Vec<_> = dag + .edges + .iter() + .filter(|e| e.consumer == node.id) + .map(|e| output[&e.producer]) + .collect(); + let input = inputs + .first() + .copied() + .unwrap_or(EdgeStatistics { rows: 1, bytes: 1 }); + let width = row_bytes(&node.output_schema); + let edge = |rows: u64| EdgeStatistics { + rows, + bytes: rows * width, + }; + let unary = |output| UnaryEdgeStatistics { + input, + output, + promql: None, + }; + let groups = |reduction: &Reduction| { + match reduction { + Reduction::Reduce(keys) if keys.keys().is_empty() && !keys.is_without() => 1, + Reduction::Reduce(keys) if !keys.is_without() => DEFAULT_GROUP_COUNT, + _ => shape.series, + } + .min(input.rows.max(1)) + }; + let (out, estimate, detail) = match &node.payload { + Payload::NonASAP(operator) => match operator { + NonASAPOp::Scan { .. } => { + // A scan reads what its time range keeps. + let lookback = dag + .edges + .iter() + .filter(|e| e.producer == node.id) + .filter_map(|e| match &nodes[&e.consumer].payload { + Payload::NonASAP(NonASAPOp::TimeRange { range, .. }) => { + Some(range.as_millis() as u64) + } + _ => None, + }) + .max() + .unwrap_or(DEFAULT_LOOKBACK_MS); + let out = edge(((shape.rows_per_ms * lookback as f64).round() as u64).max(1)); + let estimate = estimate_operator( + PhysicalOperator::Scan, + OperatorStatistics::Scan { + edges: UnaryEdgeStatistics { + input: out, + output: out, + promql: None, + }, + source_read_bytes: out.bytes, + }, + ); + (out, estimate, format!("scan {} samples", out.rows)) + } + NonASAPOp::Aggregate { + reduction, + measures, + .. + } => { + let group_count = groups(reduction); + let keys = match reduction { + Reduction::Reduce(keys) => keys.keys().len() as u64, + _ => 1, + }; + let out = edge(group_count); + let estimate = estimate_operator( + PhysicalOperator::HashAggregate { + grouping_key_count: keys, + accumulator_count: measures.len().max(1) as u64, + }, + OperatorStatistics::HashAggregate { + edges: unary(out), + group_count, + key_bytes: 16 * keys, + accumulator_bytes_per_group: 8, + }, + ); + ( + out, + estimate, + format!( + "hash aggregate {} rows into {group_count} groups", + input.rows + ), + ) + } + NonASAPOp::Sort { + keys, partition_by, .. + } => { + let partitions = if partition_by.keys().is_empty() { + 1 + } else { + DEFAULT_GROUP_COUNT.min(input.rows.max(1)) + }; + let estimate = estimate_operator( + PhysicalOperator::InMemoryComparisonSort { + ordering_key_count: keys.len() as u64, + partitioned: partitions > 1, + }, + OperatorStatistics::InMemoryComparisonSort { + edges: unary(input), + input_partitioning: split(input, partitions), + }, + ); + ( + input, + estimate, + format!("sort {} rows in {partitions} partitions", input.rows), + ) + } + NonASAPOp::Limit { + n, + offset, + partition_by, + .. + } => { + let partitions = if partition_by.keys().is_empty() { + 1 + } else { + DEFAULT_GROUP_COUNT + }; + let limit = n.map_or(u64::MAX, |n| (n as u64).saturating_mul(partitions)); + let offset = (*offset as u64).saturating_mul(partitions); + let out = edge(input.rows.saturating_sub(offset).min(limit)); + let estimate = estimate_operator( + PhysicalOperator::Limit { limit, offset }, + OperatorStatistics::Limit { edges: unary(out) }, + ); + (out, estimate, format!("limit to {} rows", out.rows)) + } + other => { + let estimate = estimate_operator( + PhysicalOperator::PassThrough, + OperatorStatistics::PassThrough { + edges: unary(input), + }, + ); + let name = match other { + NonASAPOp::TimeRange { range, .. } => format!("time range {range:?}"), + _ => "operator".into(), + }; + (input, estimate, format!("{name}: pass {} rows", input.rows)) + } + }, + Payload::ASAP(ASAPOp::SummaryAgg { + family, reduction, .. + }) => { + let group_count = groups(reduction); + let (depth, state_bytes) = summary_shape(family); + let out = EdgeStatistics { + rows: group_count, + bytes: group_count * state_bytes, + }; + let ops = input.rows as f64 * depth as f64; + ( + out, + Ok(ResourceEstimate::new(ops, out.bytes, 0)), + format!( + "build {} into {group_count} states: {} rows x depth {depth}", + family_name(family), + input.rows + ), + ) + } + Payload::ASAP(ASAPOp::SummaryEstimate { query, .. }) => { + let per_group = match query { + SketchStatistic::TopK { k } => *k as u64, + _ => 1, + }; + let out = edge(input.rows * per_group); + ( + out, + Ok(ResourceEstimate::new(out.rows as f64, 0, 0)), + format!("estimate {} groups x {per_group}", input.rows), + ) + } + Payload::ASAP(ASAPOp::FinalizeExactAccumulator { .. }) => ( + edge(input.rows), + Ok(ResourceEstimate::new(input.rows as f64, 0, 0)), + format!("finalize {} accumulators", input.rows), + ), + _ => ( + edge(input.rows), + Ok(ResourceEstimate::new(input.rows as f64, 0, 0)), + format!("{} rows", input.rows), + ), + }; + let cost = estimate + .and_then(|estimate| estimate.calibrated_cost(&calibration)) + .map_err(|error| (node.id, error))?; + let multiplier = multipliers[&node.id]; + output.insert(node.id, out); + per_node.insert( + node.id, + NodeCost { + cost: cost * multiplier as f64, + multiplier, + detail, + }, + ); + } + Ok(CandidateCost { + total: per_node.values().map(|n| n.cost).sum(), + unit: COST_UNIT, + source: COST_SOURCE, + per_node, + }) +} + +/// `input` split as evenly as integers allow into `partitions` parts. +fn split(input: EdgeStatistics, partitions: u64) -> PartitionStatistics { + let part = |total: u64, i: u64| total / partitions + u64::from(i < total % partitions); + PartitionStatistics { + partitions: (0..partitions) + .map(|i| EdgeStatistics { + rows: part(input.rows, i), + bytes: part(input.bytes, i), + }) + .collect(), + } +} + +/// Plain values are 8 bytes, strings 16; summary columns are sized apart. +fn row_bytes(schema: &Schema) -> u64 { + schema + .fields + .iter() + .map(|f| match &f.dtype { + FieldDataType::Plain(DataType::Utf8) => 16, + _ => 8, + }) + .sum::() + .max(1) +} + +/// Update operations per input row and bytes per state. +fn summary_shape(family: &FieldDataType) -> (u64, u64) { + match family { + FieldDataType::Sketch(kind, _) => match kind.params() { + SketchParams::Cms { width, depth } | SketchParams::CountSketch { width, depth } => { + (u64::from(*depth), 8 * u64::from(*width) * u64::from(*depth)) + } + SketchParams::CmsWithHeap { + width, + depth, + heap_size, + } + | SketchParams::CountSketchWithHeap { + width, + depth, + heap_size, + } => ( + u64::from(*depth) + 1, + 8 * u64::from(*width) * u64::from(*depth) + 24 * u64::from(*heap_size), + ), + _ => (1, 1_024), + }, + _ => (1, 8), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::accuracy::AccuracyEvidenceProvider; + use crate::physical_asap_optimization::optimize_physical_asap; + use crate::test_support::lower_promql; + use asap_types::ir::ExprSemantics; + use asap_types::post_asap::{BoundExpr, ErrorMetric, ProbabilityExpr}; + use asap_types::pre_asap::expr_ir::ArithmeticOpKind; + use asap_types::workload::{Evidence, Rate}; + + fn data() -> DataWorkload { + DataWorkload { + ingestion_rate: Evidence { + value: Some(Rate(10_000.0)), + ..Default::default() + }, + input_cardinality: Evidence { + value: Some(10_000), + ..Default::default() + }, + ..Default::default() + } + } + + /// Exact (P1), CMS+heap (P2) and CountSketch+heap (P3) realizations of + /// one approximate top-k query, in Pass 1 catalog order. + fn candidates() -> Vec { + let root = lower_promql( + "topk by (job) (10, sum_over_time(m[1m]))", + AccuracyTarget::EpsilonDelta { + epsilon: 0.01, + delta: 0.001, + }, + ); + let inventory = crate::per_computation_candidates::generate_per_computation_candidates( + vec![(0, QueryRoot::Operator(root))], + ) + .unwrap(); + let topk = inventory + .targets + .iter() + .position(|t| t.choices.candidates.len() > 2) + .unwrap(); + [0, 1, 2] + .into_iter() + .map(|index| { + let mut choice = vec![0; inventory.targets.len()]; + choice[topk] = index; + let roots: Vec<_> = crate::per_computation_candidates::compose_logical_candidate( + &inventory, &choice, + ) + .unwrap() + .into_iter() + .map(|(_, root)| root) + .collect(); + let mut candidate = optimize_physical_asap("L", &roots).unwrap(); + candidate.id = format!("P{}", index + 1); + candidate + }) + .collect() + } + + /// A summary whose analytical guarantee misses the target is rejected as + /// invalid with a reason; the exact plan is then selected. (Top-K + /// estimates carry no guarantee since #575, so a quantile summary is used.) + #[test] + fn accuracy_failing_candidate_is_rejected_as_invalid() { + let planned = AccuracyTarget::EpsilonDelta { + epsilon: 0.01, + delta: 0.001, + }; + let candidates = all_candidates("quantile_over_time(0.99, m[1m])", &planned); + let strict = AccuracyTarget::EpsilonDelta { + epsilon: 1e-6, + delta: 1e-9, + }; + let summaries: Vec<_> = candidates + .iter() + .filter(|c| has_summary_estimate(c)) + .map(|c| c.id.as_str()) + .collect(); + assert!(!summaries.is_empty()); + let selection = select_plan( + &candidates, + &[Some(strict)], + &data(), + PlanningModels::builtin(), + ) + .unwrap(); + assert!(!summaries.contains(&selection.selected.as_str())); + for id in &summaries { + let rejection = selection.rejected.iter().find(|r| r.id == *id).unwrap(); + assert!(!rejection.valid); + assert!( + rejection.reason.contains("misses target"), + "{}", + rejection.reason + ); + assert!(!selection.costs.contains_key(*id)); + } + } + + /// For a query with a target, Count-Min over weights not proven + /// non-negative is invalid. + #[test] + fn count_min_over_signed_weights_is_rejected_as_invalid() { + let target = AccuracyTarget::EpsilonDelta { + epsilon: 0.01, + delta: 0.001, + }; + let selection = select_plan( + &candidates(), + &[Some(target)], + &data(), + PlanningModels::builtin(), + ) + .unwrap(); + let p2 = selection.rejected.iter().find(|r| r.id == "P2").unwrap(); + assert!(!p2.valid); + assert!(p2.reason.contains("non-negative"), "{}", p2.reason); + } + + /// A query without a target constrains nothing: the Count-Min + /// candidate is admitted and priced. + #[test] + fn count_min_is_admitted_without_a_target() { + let selection = + select_plan(&candidates(), &[None], &data(), PlanningModels::builtin()).unwrap(); + assert!(selection.costs.contains_key("P2")); + assert!(selection.rejected.iter().all(|r| r.valid)); + } + + /// Each node is priced under its DAG id, and the total is the sum; + /// every candidate is either selected or rejected as costlier. The + /// CountSketch+heap estimate has no analytical guarantee, so deployment + /// evidence admits it. + #[test] + fn per_node_costs_cover_the_dag_and_sum_to_the_total() { + let candidates = candidates(); + let target = AccuracyTarget::EpsilonDelta { + epsilon: 0.01, + delta: 0.001, + }; + let selection = select_plan( + &candidates, + &[Some(target)], + &data(), + PlanningModels::builtin().with_evidence(&EveryOutputMeasuredExact), + ) + .unwrap(); + for candidate in candidates.iter().filter(|c| c.id != "P2") { + let cost = &selection.costs[&candidate.id]; + let ids: Vec<_> = candidate.dag.nodes.iter().map(|n| n.id).collect(); + let mut keys: Vec<_> = cost.per_node.keys().copied().collect(); + keys.sort(); + let mut sorted = ids.clone(); + sorted.sort(); + assert_eq!(keys, sorted); + let sum: f64 = cost.per_node.values().map(|n| n.cost).sum(); + assert_eq!(cost.total, sum); + assert!(cost.total > 0.0); + } + let costlier: Vec<_> = selection.rejected.iter().filter(|r| r.valid).collect(); + assert_eq!(costlier.len(), 1); + assert_ne!(costlier[0].id, selection.selected); + } + + /// Query 1 is `sum(...)` over query 2's output. Every node reached by + /// both queries is `NotMaterialized`, so recomputed for each: charged + /// twice. Materializing query 2's root (`MatAtQuery`) charges it once, + /// and the nodes below it once, since they run only when it does. + #[test] + fn not_materialized_shared_nodes_are_charged_per_query() { + let outer = lower_promql("sum(sum_over_time(m[1m]))", AccuracyTarget::Exact); + let inner = outer.children()[0].clone(); + let mut candidate = optimize_physical_asap( + "L", + &[QueryRoot::Operator(outer), QueryRoot::Operator(inner)], + ) + .unwrap(); + let [outer, inner] = candidate.dag.roots[..] else { + panic!("two roots") + }; + let cost = price(&candidate.dag, &data()).unwrap(); + let multipliers: BTreeMap<_, _> = cost + .per_node + .iter() + .map(|(id, c)| (*id, c.multiplier)) + .collect(); + assert_eq!(multipliers[&outer], 1); + assert_eq!(multipliers[&inner], 2); + assert!(multipliers.len() > 2, "nodes below the shared root"); + for (id, times) in &multipliers { + if *id != outer { + assert_eq!(*times, 2, "node {id} is read by both queries"); + } + } + + let node = candidate + .dag + .nodes + .iter_mut() + .find(|n| n.id == inner) + .unwrap(); + node.materialization = Materialization::MatAtQuery(asap_types::ir::Medium::Memory); + let stored = price(&candidate.dag, &data()).unwrap(); + for (id, c) in &stored.per_node { + assert_eq!(c.multiplier, 1, "node {id} is computed once"); + assert_eq!(c.cost, cost.per_node[id].cost / multipliers[id] as f64); + } + assert!(stored.total < cost.total); + } + + /// Every combination of Pass 1 candidates of one query, as Stage 2 + /// candidates `P1`, `P2`, ... + fn all_candidates(query: &str, target: &AccuracyTarget) -> Vec { + let root = lower_promql(query, target.clone()); + let inventory = crate::per_computation_candidates::generate_per_computation_candidates( + vec![(0, QueryRoot::Operator(root))], + ) + .unwrap(); + let mut choices = vec![vec![]]; + for target in &inventory.targets { + choices = choices + .into_iter() + .flat_map(|choice: Vec| { + (0..target.choices.candidates.len()).map(move |index| { + let mut choice = choice.clone(); + choice.push(index); + choice + }) + }) + .collect(); + } + choices + .iter() + .enumerate() + .map(|(index, choice)| { + let roots: Vec<_> = crate::per_computation_candidates::compose_logical_candidate( + &inventory, choice, + ) + .unwrap() + .into_iter() + .map(|(_, root)| root) + .collect(); + let mut candidate = optimize_physical_asap("L", &roots).unwrap(); + candidate.id = format!("P{}", index + 1); + candidate + }) + .collect() + } + + fn has_summary_estimate(candidate: &PhysicalCandidate) -> bool { + candidate + .dag + .nodes + .iter() + .any(|n| matches!(n.payload, Payload::ASAP(ASAPOp::SummaryEstimate { .. }))) + } + + /// Deployment evidence that every output was measured exact. + struct EveryOutputMeasuredExact; + + impl AccuracyEvidenceProvider for EveryOutputMeasuredExact { + fn output_guarantee(&self, _output: &QueryRoot) -> Option { + Some(ResultGuarantee::exact("measured")) + } + } + + /// A difference of two quantile estimates has no output guarantee, though + /// each estimate is certified: its summary candidates stay listed as + /// invalid with a reason. The deployment's empirical accuracy for the + /// output admits them. + #[test] + fn combined_estimates_need_evidence_about_the_output() { + let target = AccuracyTarget::EpsilonDelta { + epsilon: 0.01, + delta: 0.001, + }; + let candidates = all_candidates( + "quantile_over_time(0.99, m[1m]) - quantile_over_time(0.5, m[1m])", + &target, + ); + let summaries: Vec<_> = candidates + .iter() + .filter(|c| has_summary_estimate(c)) + .map(|c| c.id.as_str()) + .collect(); + assert!(!summaries.is_empty()); + + let builtin = select_plan( + &candidates, + &[Some(target.clone())], + &data(), + PlanningModels::builtin(), + ) + .unwrap(); + for id in &summaries { + let rejection = builtin.rejected.iter().find(|r| r.id == *id).unwrap(); + assert!(!rejection.valid); + assert!( + rejection.reason.contains("no guarantee"), + "{}", + rejection.reason + ); + } + assert!(!summaries.contains(&builtin.selected.as_str())); + + let measured = select_plan( + &candidates, + &[Some(target)], + &data(), + PlanningModels::builtin().with_evidence(&EveryOutputMeasuredExact), + ) + .unwrap(); + for id in &summaries { + assert!(measured.costs.contains_key(*id), "{id} is valid"); + } + } + + /// `scalar(v)` alone inherits `v`'s guarantee; any other scalar + /// expression over an approximate operand has none until #669; one over + /// exact operands only is exact. + #[test] + fn scalar_of_one_vector_inherits_its_guarantee_and_nothing_else_does() { + let node = lower_promql("sum(m)", AccuracyTarget::Exact); + let approximate = ResultGuarantee { + metric: ErrorMetric::AbsoluteValue, + bound: BoundExpr::Constant { value: 0.01 }, + failure_probability: ProbabilityExpr::Constant { value: 0.001 }, + provenance: vec![], + }; + let with = |guarantee: Option| { + ScalarExpr::PromqlScalarFromVector(Rc::new((*node).clone().with_guarantee(guarantee))) + }; + let doubled = |operand: ScalarExpr| { + QueryRoot::Scalar(ScalarExpr::Arithmetic { + op: ArithmeticOpKind::Mul, + left: Box::new(operand), + right: Box::new(ScalarExpr::literal_f64(2.0)), + semantics: ExprSemantics::Promql, + }) + }; + assert_eq!( + root_guarantee(&QueryRoot::Scalar(with(Some(approximate.clone())))), + Some(approximate.clone()) + ); + assert_eq!(root_guarantee(&doubled(with(Some(approximate)))), None); + assert!( + root_guarantee(&doubled(with(Some(ResultGuarantee::exact("exact"))))) + .is_some_and(|g| g.is_exact()) + ); + } +} diff --git a/crates/devtools/src/bin/stage_pipeline.rs b/crates/devtools/src/bin/stage_pipeline.rs index 23fe0d06..dab589cc 100644 --- a/crates/devtools/src/bin/stage_pipeline.rs +++ b/crates/devtools/src/bin/stage_pipeline.rs @@ -4,15 +4,25 @@ // --promql "topk by (job) (10, rate(x[1m]))" --epsilon 0.01 --delta 0.001 --out run.json // // Writes an `asap-stage-pipeline/v1` document (tools/dag-viewer) with the -// first two planner stages only: +// four planner stages (#509 MVP): // - language_specific_frontends: the frontends' workload DAG, one root per query; // - logical_asap_optimization: Pass 1 whole-workload candidates, one per // choice of a per-computation candidate for every target (the Cartesian // product), in enumeration order and capped by `--max-candidates` // (default 64). A candidate that cannot be composed carries `error` -// instead of `dag`. -// Physical ASAP-aware optimization and plan selection, and therefore cost, -// are absent. +// instead of `dag`; +// - physical_asap_optimization: one physical candidate per composed logical +// candidate (operator implementation; every sub-DAG NotMaterialized), no +// cost; a composed logical candidate Stage 2 cannot implement is listed +// under `pruned` with the reason; +// - plan_selection: per-candidate costs, the selected candidate, and +// every other candidate as rejected (`valid: false`) or costlier. Each +// node's `cost` is one computation's price times `multiplier`, the +// times it is computed: once if materialized, else once per query (or +// materialized node) that reads it, so a shared NotMaterialized node is +// charged once per query. When +// selection fails, `selected` is null, with the rejections, or with +// `error` for any other failure. // // `--promql` may repeat. `--epsilon`/`--delta` apply to every `--promql` // query; without them the queries are exact. `--interval-ms` is the source @@ -24,7 +34,11 @@ use asap_aware_mapping::per_computation_candidates::{ compose_logical_candidate, generate_per_computation_candidates, ComputationCandidate, PerComputationCandidates, }; +use asap_aware_mapping::physical_asap_optimization::optimize_physical_asap; +use asap_aware_mapping::plan_selection::{select_plan, Rejection, Selection, SelectionError}; +use asap_aware_mapping::PlanningModels; use asap_types::ir::flat::{flatten, FlatDag}; +use asap_types::ir::schema_support::with_promql_series_identity; use asap_types::ir::{OperatorNode, QueryRoot}; use asap_types::post_asap::SketchAlgorithm; use asap_types::types::AccuracyTarget; @@ -88,8 +102,18 @@ fn run(args: Vec) -> Result<(), String> { } fn stage_pipeline(workload: &PlanningWorkload, max_candidates: usize) -> Result { + // PromQL rows carry each series' full identity as a column: the row + // representation per-series state needs at runtime. let roots = asap_frontend_promql::lower_promql_query_workload(workload, 0) - .map_err(|e| format!("lowering: {e}"))?; + .map_err(|e| format!("lowering: {e}"))? + .into_iter() + .map(|root| match root { + QueryRoot::Operator(node) => with_promql_series_identity(&node) + .map(QueryRoot::Operator) + .map_err(|e| format!("series identity: {e}")), + scalar => Ok(scalar), + }) + .collect::, _>>()?; let frontends_dag = export(&roots)?; let inventory = generate_per_computation_candidates(roots.into_iter().enumerate().collect()) .map_err(|e| format!("Pass 1: {e}"))?; @@ -100,18 +124,31 @@ fn stage_pipeline(workload: &PlanningWorkload, max_candidates: usize) -> Result< .product::(); let owners = target_owners(&inventory); let mut candidates = Vec::new(); + let mut physical = Vec::new(); + let mut pruned = Vec::new(); let mut choice = vec![0; inventory.targets.len()]; for index in 0..combinations.min(max_candidates) { - let mut candidate = json!({ - "id": format!("L{}", index + 1), - "label": label(&inventory, &owners, &choice), - }); + let (id, label) = ( + format!("L{}", index + 1), + label(&inventory, &owners, &choice), + ); + let mut candidate = json!({ "id": id, "label": label }); // A choice that composition cannot substitute yet is listed with its // error rather than ending the run. match compose_logical_candidate(&inventory, &choice) { Ok(roots) => { let roots: Vec<_> = roots.into_iter().map(|(_, root)| root).collect(); candidate["dag"] = json!(export(&roots)?); + match optimize_physical_asap(&id, &roots) { + Ok(mut physical_candidate) => { + physical_candidate.id = format!("P{}", index + 1); + physical_candidate.label = label; + physical.push(physical_candidate); + } + Err(e) => pruned.push( + json!({ "from_logical": id, "label": label, "reason": e.to_string() }), + ), + } } Err(error) => candidate["error"] = json!(error.to_string()), } @@ -125,6 +162,17 @@ fn stage_pipeline(workload: &PlanningWorkload, max_candidates: usize) -> Result< *digit = 0; } } + let targets: Vec<_> = workload + .query_workload + .entries() + .map(|entry| Some(entry.requirements.accuracy.target())) + .collect(); + let data = workload.data_workload.clone().unwrap_or_default(); + let selection = select_plan(&physical, &targets, &data, PlanningModels::builtin()); + let physical_json: Vec<_> = physical + .iter() + .map(|p| json!({ "id": p.id, "from_logical": p.from_logical, "label": p.label, "dag": p.dag })) + .collect(); Ok(json!({ "format": "asap-stage-pipeline/v1", "workload": { "queries": workload_queries(workload) }, @@ -134,9 +182,55 @@ fn stage_pipeline(workload: &PlanningWorkload, max_candidates: usize) -> Result< "capped": combinations > max_candidates, "candidates": candidates, }, + "physical_asap_optimization": { "candidates": physical_json, "pruned": pruned }, + "plan_selection": plan_selection_json(&selection), })) } +fn plan_selection_json(selection: &Result) -> Value { + let selection = match selection { + Ok(selection) => selection, + Err(SelectionError::NoValidCandidate(rejected)) => { + return json!({ "costs": {}, "selected": null, "rejected": rejections_json(rejected) }) + } + Err(error) => { + return json!({ "costs": {}, "selected": null, "rejected": [], "error": error.to_string() }) + } + }; + let costs: serde_json::Map<_, _> = selection + .costs + .iter() + .map(|(id, cost)| { + let per_node: serde_json::Map<_, _> = cost + .per_node + .iter() + .map(|(node, c)| { + ( + node.to_string(), + json!({ "cost": c.cost, "multiplier": c.multiplier, "detail": c.detail }), + ) + }) + .collect(); + ( + id.clone(), + json!({ "total": cost.total, "unit": cost.unit, "source": cost.source, "per_node": per_node }), + ) + }) + .collect(); + json!({ + "costs": costs, + "selected": selection.selected, + "rejected": rejections_json(&selection.rejected), + }) +} + +fn rejections_json(rejected: &[Rejection]) -> Vec { + rejected + .iter() + .map(|r| json!({ "id": r.id, "valid": r.valid, "reason": r.reason })) + .collect() +} + /// A logical DAG as a flat node list: node `i` is `nodes[i]`, and children are /// node indices inside each operator. fn export(roots: &[QueryRoot]) -> Result { @@ -338,3 +432,39 @@ fn planner_layering_example1() -> PlanningWorkload { }), } } + +#[cfg(test)] +mod tests { + use super::*; + + /// With no valid candidate, the document is still written: `selected` + /// is null and every rejection is listed. + #[test] + fn failed_selection_lists_rejections_without_a_selection() { + let rejected = vec![Rejection { + id: "P1".into(), + valid: false, + reason: "q1: misses target".into(), + }]; + let document = plan_selection_json(&Err(SelectionError::NoValidCandidate(rejected))); + assert_eq!( + document, + json!({ + "costs": {}, + "selected": null, + "rejected": [{ "id": "P1", "valid": false, "reason": "q1: misses target" }], + }) + ); + } + + /// A run with no physical candidate (here, none enumerated) still emits + /// every stage, with no selection. + #[test] + fn run_without_candidates_emits_plan_selection() { + let workload = promql_batch(&["sum(m)".into()], AccuracyTarget::Exact, 15_000); + let document = stage_pipeline(&workload, 0).unwrap(); + assert_eq!(document["physical_asap_optimization"]["pruned"], json!([])); + assert_eq!(document["plan_selection"]["selected"], Value::Null); + assert_eq!(document["plan_selection"]["rejected"], json!([])); + } +} diff --git a/crates/devtools/tests/stage_pipeline.rs b/crates/devtools/tests/stage_pipeline.rs index ef5a688b..897ab193 100644 --- a/crates/devtools/tests/stage_pipeline.rs +++ b/crates/devtools/tests/stage_pipeline.rs @@ -1,11 +1,10 @@ -//! `stage_pipeline` writes a valid viewer document for #509 Example 1: the -//! language-specific frontends' DAG and the logical ASAP-aware optimization -//! candidates. +//! `stage_pipeline` writes a valid four-stage viewer document for #509 Example 1. use std::collections::HashSet; use std::path::PathBuf; use std::process::Command; use asap_types::ir::flat::FlatDag; +use asap_types::ir::physical_export::{PhysicalASAPDAG, PhysicalASAPDAGDocument}; use asap_types::ir::QueryRoot; use serde_json::Value; @@ -47,9 +46,11 @@ fn validated_dag(value: &Value, queries: usize) { } } -/// Every DAG is a well-formed flat DAG with one root per query; ids are unique; -/// only the frontends and logical ASAP-aware optimization are present; the -/// committed fixture is current. +/// Every logical DAG is a well-formed flat DAG and every physical DAG validates; +/// each has one root per query; ids are unique; Stage 2 implements or prunes +/// (with a reason) each composed logical candidate, with no cost; Stage 3 +/// accounts for every Stage 2 candidate once and costs exactly the valid ones; +/// the committed fixture is current. #[test] fn example1_document_is_valid_and_committed_fixture_is_current() { let document = generate(&[]); @@ -63,8 +64,6 @@ fn example1_document_is_valid_and_committed_fixture_is_current() { assert_eq!(document["format"], "asap-stage-pipeline/v1"); let queries = document["workload"]["queries"].as_array().unwrap().len(); assert_eq!(queries, 2); - assert!(document.get("physical_asap_optimization").is_none()); - assert!(document.get("plan_selection").is_none()); validated_dag(&document["language_specific_frontends"]["dag"], queries); let logical = &document["logical_asap_optimization"]; @@ -72,10 +71,55 @@ fn example1_document_is_valid_and_committed_fixture_is_current() { assert_eq!(logical["capped"], false); assert_eq!(logical["combinations"], candidates.len()); let mut ids = HashSet::new(); + let mut composed = HashSet::new(); for candidate in candidates { - assert!(ids.insert(candidate["id"].as_str().unwrap())); - validated_dag(&candidate["dag"], queries); + let id = candidate["id"].as_str().unwrap(); + assert!(ids.insert(id)); + if candidate.get("dag").is_some() { + composed.insert(id); + validated_dag(&candidate["dag"], queries); + } + } + + let physical = document["physical_asap_optimization"]["candidates"] + .as_array() + .unwrap(); + let pruned = document["physical_asap_optimization"]["pruned"] + .as_array() + .unwrap(); + assert!(pruned + .iter() + .all(|p| !p["reason"].as_str().unwrap().is_empty())); + let sources: HashSet<_> = physical + .iter() + .chain(pruned) + .map(|p| p["from_logical"].as_str().unwrap()) + .collect(); + assert_eq!(physical.len() + pruned.len(), composed.len()); + assert_eq!(sources, composed); + for candidate in physical { + assert!(candidate.get("cost").is_none(), "Stage 2 has no cost"); + let dag: PhysicalASAPDAG = serde_json::from_value(candidate["dag"].clone()).unwrap(); + assert_eq!(dag.roots.len(), queries); + PhysicalASAPDAGDocument::new(dag).validate().unwrap(); + } + + let selection = &document["plan_selection"]; + let costs = selection["costs"].as_object().unwrap(); + let selected = selection["selected"].as_str().unwrap(); + let mut accounted = HashSet::from([selected]); + for rejection in selection["rejected"].as_array().unwrap() { + let id = rejection["id"].as_str().unwrap(); + assert!(accounted.insert(id), "{id} accounted for once"); + assert!(!rejection["reason"].as_str().unwrap().is_empty()); + assert_eq!( + rejection["valid"].as_bool().unwrap(), + costs.contains_key(id) + ); } + assert!(costs.contains_key(selected)); + let all: HashSet<_> = physical.iter().map(|p| p["id"].as_str().unwrap()).collect(); + assert_eq!(accounted, all); } /// The Cartesian product is cut at `--max-candidates` and says so. diff --git a/crates/integration-tests/tests/physical_asap_optimization_sql.rs b/crates/integration-tests/tests/physical_asap_optimization_sql.rs new file mode 100644 index 00000000..020d87fb --- /dev/null +++ b/crates/integration-tests/tests/physical_asap_optimization_sql.rs @@ -0,0 +1,189 @@ +//! SQL heavy-hitter queries through Stage 1 (every Pass 1 choice composed) +//! and physical ASAP-aware optimization: exact top-k sorts by the ranked +//! measure, descending, whatever the group keys are called or typed, and no +//! valid candidate is pruned. + +use asap_aware_mapping::per_computation_candidates::{ + compose_logical_candidate, generate_per_computation_candidates, ComputationCandidate, +}; +use asap_aware_mapping::physical_asap_optimization::optimize_physical_asap; +use asap_frontend_sql::{lower_sql, SqlCatalog}; +use asap_types::ir::{NonASAPOp, OperatorNode, QueryRoot, ScalarExpr}; +use asap_types::pre_asap::schema::{DataType, Field, Schema}; +use asap_types::pre_asap::AggIntent; +use asap_types::types::AccuracyTarget; + +fn approximate() -> AccuracyTarget { + AccuracyTarget::EpsilonDelta { + epsilon: 0.01, + delta: 0.001, + } +} + +fn names(node: &OperatorNode) -> Vec { + node.schema.fields.iter().map(|f| f.name.clone()).collect() +} + +/// The `(sort column name, ascending)` of an exact top-k's `Sort`, if `root` +/// is one (a heavy-hitter summary candidate has none). +fn exact_topk_sort(root: &OperatorNode) -> Option<(String, bool)> { + let NonASAPOp::Project { child: limit, .. } = root.non_asap()? else { + return None; + }; + let NonASAPOp::Limit { + n: Some(5), + child: sort, + .. + } = limit.non_asap()? + else { + return None; + }; + let NonASAPOp::Sort { keys, .. } = sort.expect_non_asap() else { + panic!("sort below limit") + }; + let [key] = keys.as_slice() else { + panic!("one sort key") + }; + let ScalarExpr::Column(column) = key.expr else { + panic!("sort by a column") + }; + Some((sort.schema.fields[column].name.clone(), key.ascending)) +} + +/// Every Pass 1 choice of `sql` under `accuracy`, composed and physically +/// optimized: each must succeed, every exact top-k must sort by `ranked` +/// descending, and every root must output `root_names`. Returns how many +/// candidates had an exact top-k. +async fn check_every_choice( + sql: &str, + accuracy: AccuracyTarget, + ranked: &str, + root_names: &[&str], +) -> usize { + let catalog = SqlCatalog::new().with_table( + "metrics", + Schema::new(vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + Field::plain("host", DataType::Utf8, false), + Field::plain("port", DataType::Int64, false), + ]), + ); + let root = lower_sql(sql, &catalog, accuracy).await.unwrap(); + let NonASAPOp::Aggregate { measures, .. } = root.expect_non_asap() else { + panic!("{sql}: heavy-hitter aggregate root") + }; + assert!( + matches!(measures.as_slice(), [AggIntent::TopK { k: 5, .. }]), + "{sql}" + ); + let inventory = + generate_per_computation_candidates(vec![(0, QueryRoot::Operator(root))]).unwrap(); + // The cartesian product of every target's candidates. + let mut choices = vec![vec![]]; + for target in &inventory.targets { + choices = choices + .into_iter() + .flat_map(|prefix: Vec| { + (0..target.choices.candidates.len()).map(move |i| { + let mut choice = prefix.clone(); + choice.push(i); + choice + }) + }) + .collect(); + } + let mut exact = 0; + for choice in choices { + let composed = match compose_logical_candidate(&inventory, &choice) { + Ok(composed) => composed, + // Stage 1 cannot yet compose a heavy-hitter summary over some + // keys (an integer or several key columns); Stage 2 never sees + // that candidate. Every other choice must compose. + Err(e) => { + let summarized_topk = inventory.targets.iter().zip(&choice).any(|(t, &i)| { + let NonASAPOp::Aggregate { measures, .. } = t.target.expect_non_asap() else { + return false; + }; + matches!(measures[t.measure], AggIntent::TopK { .. }) + && t.choices.candidates[i] != ComputationCandidate::Exact + }); + assert!(summarized_topk, "{sql}, choice {choice:?}: {e}"); + continue; + } + }; + let roots: Vec<_> = composed.into_iter().map(|(_, root)| root).collect(); + let candidate = optimize_physical_asap("L1", &roots) + .unwrap_or_else(|e| panic!("{sql}, choice {choice:?} pruned: {e}")); + let QueryRoot::Operator(physical) = &candidate.roots[0] else { + panic!("operator root") + }; + assert_eq!(names(physical), root_names, "{sql}, choice {choice:?}"); + if let Some(sort) = exact_topk_sort(physical) { + assert_eq!( + sort, + (ranked.to_string(), false), + "{sql}, choice {choice:?}" + ); + exact += 1; + } + } + exact +} + +/// Each query under an exact and an approximate target; every exact top-k +/// sorts by the ranked measure. +async fn check(sql: &str, ranked: &str, root_names: &[&str]) { + for accuracy in [AccuracyTarget::Exact, approximate()] { + let exact = check_every_choice(sql, accuracy.clone(), ranked, root_names).await; + assert!( + exact > 0, + "{sql} under {accuracy:?}: an exact top-k candidate" + ); + } +} + +/// Grouping by a column named `value` still ranks by the count. +#[tokio::test] +async fn group_key_named_value_ranks_by_count() { + check( + "SELECT value, COUNT(*) AS c FROM metrics GROUP BY value ORDER BY c DESC LIMIT 5", + "count(Int64(1))", + &["topk_5"], + ) + .await; +} + +/// An integer group key leaves no `value` column; the count is still ranked. +#[tokio::test] +async fn integer_group_key_ranks_by_count() { + check( + "SELECT port, COUNT(*) AS c FROM metrics GROUP BY port ORDER BY c DESC LIMIT 5", + "count(Int64(1))", + &["topk_5"], + ) + .await; +} + +/// Two group keys, one of them an integer. +#[tokio::test] +async fn multi_key_ranks_by_count() { + check( + "SELECT host, port, COUNT(*) AS c FROM metrics GROUP BY host, port \ + ORDER BY c DESC LIMIT 5", + "count(Int64(1))", + &["topk_5"], + ) + .await; +} + +/// A SUM ranking sorts by the sum. +#[tokio::test] +async fn sum_ranking_ranks_by_sum() { + check( + "SELECT port, SUM(value) AS s FROM metrics GROUP BY port ORDER BY s DESC LIMIT 5", + "sum(metrics.value)", + &["topk_5"], + ) + .await; +} diff --git a/docs/user_guide_docs/stage-viewer.md b/docs/user_guide_docs/stage-viewer.md index 04b79612..ff324c19 100644 --- a/docs/user_guide_docs/stage-viewer.md +++ b/docs/user_guide_docs/stage-viewer.md @@ -116,7 +116,11 @@ plans: ### Stage 3 · plans by cost -Every physical plan, cheapest first, with its total cost per second: +Every physical plan, cheapest first, with its total cost per second. +A node that is not materialized is recomputed for each query that reads it, +so a node shared by two queries is charged twice; a materialized node, at +ingestion or at query time, is charged once, and the nodes below it once for +it (plus once for each query that also reads them directly). - **✓ selected**: the plan ASAPPlanner chooses. - **valid · costlier**: meets every requirement, but costs more. diff --git a/tools/dag-viewer/examples/planner-layering-example1.json b/tools/dag-viewer/examples/planner-layering-example1.json index 34a2b9f1..6218ff9c 100644 --- a/tools/dag-viewer/examples/planner-layering-example1.json +++ b/tools/dag-viewer/examples/planner-layering-example1.json @@ -11,7 +11,7 @@ "Scan": { "predicates": [], "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -36,6 +36,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -51,7 +59,7 @@ }, "result_kind": "instant_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -76,6 +84,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -100,7 +116,7 @@ }, "result_kind": "range_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -125,6 +141,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -155,7 +179,7 @@ }, "result_kind": "instant_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -180,6 +204,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -251,7 +283,7 @@ "Scan": { "predicates": [], "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -276,6 +308,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -291,7 +331,7 @@ }, "result_kind": "instant_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -316,6 +356,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -340,7 +388,7 @@ }, "result_kind": "range_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -365,6 +413,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -396,7 +452,7 @@ }, "result_kind": "instant_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -421,6 +477,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -526,7 +590,7 @@ "Scan": { "predicates": [], "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -551,6 +615,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -566,7 +638,7 @@ }, "result_kind": "instant_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -591,6 +663,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -629,7 +709,7 @@ }, "result_kind": "range_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -654,6 +734,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -691,7 +779,7 @@ }, "result_kind": "state", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -719,6 +807,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -752,7 +848,7 @@ }, "result_kind": "instant_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -777,6 +873,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -925,7 +1029,7 @@ "Scan": { "predicates": [], "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -950,6 +1054,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -965,7 +1077,7 @@ }, "result_kind": "instant_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -990,6 +1102,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -1028,7 +1148,7 @@ }, "result_kind": "range_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -1053,6 +1173,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -1090,7 +1218,7 @@ }, "result_kind": "state", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -1118,6 +1246,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -1151,7 +1287,7 @@ }, "result_kind": "instant_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -1176,6 +1312,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -1295,7 +1439,7 @@ "Scan": { "predicates": [], "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -1320,6 +1464,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -1335,7 +1487,7 @@ }, "result_kind": "instant_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -1360,6 +1512,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -1398,7 +1558,7 @@ }, "result_kind": "range_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -1423,6 +1583,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -1460,7 +1628,7 @@ }, "result_kind": "state", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -1488,6 +1656,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -1521,7 +1697,7 @@ }, "result_kind": "instant_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -1546,6 +1722,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -1694,7 +1878,7 @@ "Scan": { "predicates": [], "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -1719,6 +1903,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -1734,7 +1926,7 @@ }, "result_kind": "instant_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -1759,6 +1951,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -1797,7 +1997,7 @@ }, "result_kind": "range_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -1822,6 +2022,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -1859,7 +2067,7 @@ }, "result_kind": "state", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -1887,6 +2095,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -1920,7 +2136,7 @@ }, "result_kind": "instant_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -1945,6 +2161,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -1979,14 +2203,8 @@ "grouping": "PerSubpopulationInstance", "input": { "item": { - "EntityIdentity": { - "PromqlLabelSet": { - "excluding": [ - { - "Named": "job" - } - ] - } + "Column": { + "Named": "$promql_series_identity" } }, "weight": { @@ -2185,7 +2403,7 @@ "Scan": { "predicates": [], "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -2210,6 +2428,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -2225,7 +2451,7 @@ }, "result_kind": "instant_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -2250,6 +2476,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -2288,7 +2522,7 @@ }, "result_kind": "range_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -2313,6 +2547,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -2350,7 +2592,7 @@ }, "result_kind": "state", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -2378,6 +2620,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -2411,7 +2661,7 @@ }, "result_kind": "instant_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -2436,11 +2686,19 @@ "name": "job", "nullable": true, "table": null - } - ], - "time_index": 0, - "unique_keys": [] - }, + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, "timing": null }, { @@ -2584,7 +2842,7 @@ "Scan": { "predicates": [], "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -2609,6 +2867,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -2624,7 +2890,7 @@ }, "result_kind": "instant_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -2649,6 +2915,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -2687,7 +2961,7 @@ }, "result_kind": "range_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -2712,6 +2986,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -2749,7 +3031,7 @@ }, "result_kind": "state", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -2777,6 +3059,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -2810,7 +3100,7 @@ }, "result_kind": "instant_vector", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -2835,6 +3125,14 @@ "name": "job", "nullable": true, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], "time_index": 0, @@ -2869,14 +3167,8 @@ "grouping": "PerSubpopulationInstance", "input": { "item": { - "EntityIdentity": { - "PromqlLabelSet": { - "excluding": [ - { - "Named": "job" - } - ] - } + "Column": { + "Named": "$promql_series_identity" } }, "weight": { @@ -3054,6 +3346,4753 @@ "capped": false, "combinations": 3 }, + "physical_asap_optimization": { + "candidates": [ + { + "dag": { + "edges": [ + { + "consumer": 1, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 0, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 2, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 1, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 3, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 2, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 4, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 3, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 5, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "producer": 4, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 7, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 6, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 8, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 7, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 9, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 8, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 10, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 9, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 11, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 10, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 12, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 11, + "role": "Input", + "window": "NotApplicable" + } + ], + "nodes": [ + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 0, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Scan": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 1, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "TimeRange": { + "child": 0, + "kind": "range", + "range": { + "nanos": 0, + "secs": 60 + } + } + } + } + }, + { + "guarantee": null, + "id": 2, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 1, + "family": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": "PerEntity" + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "id": 3, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 2 + } + } + } + }, + { + "guarantee": null, + "id": 4, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 3, + "family": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": { + "Reduce": [ + 2 + ] + } + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "id": 5, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "sum", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 4 + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 6, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Scan": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 7, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "TimeRange": { + "child": 6, + "kind": "range", + "range": { + "nanos": 0, + "secs": 60 + } + } + } + } + }, + { + "guarantee": null, + "id": 8, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 7, + "family": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": "PerEntity" + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "id": 9, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 8 + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "id": 10, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Sort": { + "child": 9, + "keys": [ + { + "ascending": false, + "expr": { + "Column": 1 + }, + "nulls_first": false + } + ], + "partition_by": [ + 2 + ] + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 11, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Limit": { + "child": 10, + "n": 10, + "offset": 0, + "partition_by": [ + 2 + ] + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 12, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "topk_10", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Project": { + "child": 11, + "cols": [ + { + "alias": "job", + "expr": { + "Column": 2 + } + }, + { + "alias": "topk_10", + "expr": { + "Column": 1 + } + } + ], + "qualifier": null + } + } + } + } + ], + "roots": [ + 5, + 12 + ], + "scalar_roots": [] + }, + "from_logical": "L1", + "id": "P1", + "label": "Q1 exact (Sum acc, Rate acc) · Q2 exact (Sum acc)" + }, + { + "dag": { + "edges": [ + { + "consumer": 1, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 0, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 2, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 1, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 3, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 2, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 4, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 3, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 5, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "producer": 4, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 7, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 6, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 8, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 7, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 9, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 8, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 10, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 9, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 11, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Sketch": [ + { + "algorithm": "CmsWithHeap", + "category": "TopK", + "params": { + "CmsWithHeap": { + "depth": 7, + "heap_size": 100, + "width": 272 + } + } + }, + "PerSubpopulationInstance" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "producer": 10, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 12, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "topk", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "producer": 11, + "role": "Input", + "window": "NotApplicable" + } + ], + "nodes": [ + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 0, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Scan": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 1, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "TimeRange": { + "child": 0, + "kind": "range", + "range": { + "nanos": 0, + "secs": 60 + } + } + } + } + }, + { + "guarantee": null, + "id": 2, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 1, + "family": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": "PerEntity" + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "id": 3, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 2 + } + } + } + }, + { + "guarantee": null, + "id": 4, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 3, + "family": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": { + "Reduce": [ + 2 + ] + } + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "id": 5, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "sum", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 4 + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 6, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Scan": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 7, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "TimeRange": { + "child": 6, + "kind": "range", + "range": { + "nanos": 0, + "secs": 60 + } + } + } + } + }, + { + "guarantee": null, + "id": 8, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 7, + "family": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": "PerEntity" + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "id": 9, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 8 + } + } + } + }, + { + "guarantee": null, + "id": 10, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Sketch": [ + { + "algorithm": "CmsWithHeap", + "category": "TopK", + "params": { + "CmsWithHeap": { + "depth": 7, + "heap_size": 100, + "width": 272 + } + } + }, + "PerSubpopulationInstance" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 9, + "family": { + "Sketch": [ + { + "algorithm": "CmsWithHeap", + "category": "TopK", + "params": { + "CmsWithHeap": { + "depth": 7, + "heap_size": 100, + "width": 272 + } + } + }, + "PerSubpopulationInstance" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": { + "Column": { + "Named": "$promql_series_identity" + } + }, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": { + "Reduce": [ + 2 + ] + } + } + } + } + }, + { + "guarantee": null, + "id": 11, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "topk", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryEstimate": { + "query": { + "TopK": { + "k": 10 + } + }, + "summary_input": 10 + } + } + } + }, + { + "guarantee": null, + "id": 12, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "topk_10", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Project": { + "child": 11, + "cols": [ + { + "alias": "job", + "expr": { + "Column": 0 + } + }, + { + "alias": "topk_10", + "expr": { + "Column": 1 + } + } + ], + "qualifier": null + } + } + } + } + ], + "roots": [ + 5, + 12 + ], + "scalar_roots": [] + }, + "from_logical": "L2", + "id": "P2", + "label": "Q1 exact (Sum acc, Rate acc) · Q2 CMS+heap (Sum acc)" + }, + { + "dag": { + "edges": [ + { + "consumer": 1, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 0, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 2, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 1, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 3, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 2, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 4, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 3, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 5, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "producer": 4, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 7, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 6, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 8, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 7, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 9, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 8, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 10, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 9, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 11, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Sketch": [ + { + "algorithm": "CountSketchWithHeap", + "category": "TopK", + "params": { + "CountSketchWithHeap": { + "depth": 125, + "heap_size": 100, + "width": 30000 + } + } + }, + "PerSubpopulationInstance" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "producer": 10, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 12, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "topk", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "producer": 11, + "role": "Input", + "window": "NotApplicable" + } + ], + "nodes": [ + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 0, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Scan": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 1, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "TimeRange": { + "child": 0, + "kind": "range", + "range": { + "nanos": 0, + "secs": 60 + } + } + } + } + }, + { + "guarantee": null, + "id": 2, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 1, + "family": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": "PerEntity" + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "id": 3, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 2 + } + } + } + }, + { + "guarantee": null, + "id": 4, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 3, + "family": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": { + "Reduce": [ + 2 + ] + } + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "id": 5, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "sum", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 4 + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 6, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Scan": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 7, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "TimeRange": { + "child": 6, + "kind": "range", + "range": { + "nanos": 0, + "secs": 60 + } + } + } + } + }, + { + "guarantee": null, + "id": 8, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 7, + "family": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": "PerEntity" + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "id": 9, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 8 + } + } + } + }, + { + "guarantee": null, + "id": 10, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Sketch": [ + { + "algorithm": "CountSketchWithHeap", + "category": "TopK", + "params": { + "CountSketchWithHeap": { + "depth": 125, + "heap_size": 100, + "width": 30000 + } + } + }, + "PerSubpopulationInstance" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 9, + "family": { + "Sketch": [ + { + "algorithm": "CountSketchWithHeap", + "category": "TopK", + "params": { + "CountSketchWithHeap": { + "depth": 125, + "heap_size": 100, + "width": 30000 + } + } + }, + "PerSubpopulationInstance" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": { + "Column": { + "Named": "$promql_series_identity" + } + }, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": { + "Reduce": [ + 2 + ] + } + } + } + } + }, + { + "guarantee": null, + "id": 11, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "topk", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryEstimate": { + "query": { + "TopK": { + "k": 10 + } + }, + "summary_input": 10 + } + } + } + }, + { + "guarantee": null, + "id": 12, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "topk_10", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Project": { + "child": 11, + "cols": [ + { + "alias": "job", + "expr": { + "Column": 0 + } + }, + { + "alias": "topk_10", + "expr": { + "Column": 1 + } + } + ], + "qualifier": null + } + } + } + } + ], + "roots": [ + 5, + 12 + ], + "scalar_roots": [] + }, + "from_logical": "L3", + "id": "P3", + "label": "Q1 exact (Sum acc, Rate acc) · Q2 CountSketch+heap (Sum acc)" + } + ], + "pruned": [] + }, + "plan_selection": { + "costs": { + "P1": { + "per_node": { + "0": { + "cost": 23.2, + "detail": "scan 4000000 samples", + "multiplier": 1 + }, + "1": { + "cost": 4.0, + "detail": "time range 60s: pass 4000000 rows", + "multiplier": 1 + }, + "10": { + "cost": 14.0, + "detail": "sort 1000000 rows in 100 partitions", + "multiplier": 1 + }, + "11": { + "cost": 0.001, + "detail": "limit to 1000 rows", + "multiplier": 1 + }, + "12": { + "cost": 0.001, + "detail": "operator: pass 1000 rows", + "multiplier": 1 + }, + "2": { + "cost": 4.0, + "detail": "build exact Rate accumulator into 1000000 states: 4000000 rows x depth 1", + "multiplier": 1 + }, + "3": { + "cost": 1.0, + "detail": "finalize 1000000 accumulators", + "multiplier": 1 + }, + "4": { + "cost": 1.0, + "detail": "build exact Sum accumulator into 100 states: 1000000 rows x depth 1", + "multiplier": 1 + }, + "5": { + "cost": 0.00009999999999999999, + "detail": "finalize 100 accumulators", + "multiplier": 1 + }, + "6": { + "cost": 23.2, + "detail": "scan 4000000 samples", + "multiplier": 1 + }, + "7": { + "cost": 4.0, + "detail": "time range 60s: pass 4000000 rows", + "multiplier": 1 + }, + "8": { + "cost": 4.0, + "detail": "build exact Sum accumulator into 1000000 states: 4000000 rows x depth 1", + "multiplier": 1 + }, + "9": { + "cost": 1.0, + "detail": "finalize 1000000 accumulators", + "multiplier": 1 + } + }, + "source": "analytical-cost-v1 (illustrative statistics)", + "total": 79.40210000000002, + "unit": "cpu_ms_per_workload_evaluation" + } + }, + "rejected": [ + { + "id": "P2", + "reason": "q2: CmsWithHeap needs non-negative update weights, and these are not proven non-negative", + "valid": false + }, + { + "id": "P3", + "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target EpsilonDelta { epsilon: 0.01, delta: 0.001 }", + "valid": false + } + ], + "selected": "P1" + }, "workload": { "queries": [ {