diff --git a/Cargo.lock b/Cargo.lock index de76c170..f40752fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -388,6 +388,7 @@ name = "asap-integration-tests" version = "0.1.0" dependencies = [ "asap-aware-mapping", + "asap-devtools", "asap-frontend-promql", "asap-frontend-sql", "asap-physical-operators", diff --git a/crates/devtools/src/bin/stage_pipeline.rs b/crates/devtools/src/bin/stage_pipeline.rs index dab589cc..1018b401 100644 --- a/crates/devtools/src/bin/stage_pipeline.rs +++ b/crates/devtools/src/bin/stage_pipeline.rs @@ -1,5 +1,7 @@ // cargo run -p asap-devtools --bin stage_pipeline -- \ // --example planner-layering-1 --out planner-layering-example1.json +// (also planner-layering-3a and planner-layering-3b: #509 Example 3, +// Patterns A and B; 3a has 243 candidates, all priced, 64 shown by default) // cargo run -p asap-devtools --bin stage_pipeline -- \ // --promql "topk by (job) (10, rate(x[1m]))" --epsilon 0.01 --delta 0.001 --out run.json // @@ -8,50 +10,48 @@ // - language_specific_frontends: the frontends' workload DAG, one root per query; // - logical_asap_optimization: Pass 1 whole-workload candidates, one per // choice of a per-computation candidate for every target (the Cartesian -// product), in enumeration order and capped by `--max-candidates` -// (default 64). A candidate that cannot be composed carries `error` -// instead of `dag`; +// product), in enumeration order. Every candidate is optimized and +// priced; the document shows the first `--max-displayed` (default 64) +// and the selected one, with `combinations` (how many exist) and +// `capped` (whether some are not shown). A candidate that cannot be +// composed carries `error` instead of `dag`; // - physical_asap_optimization: one physical candidate per composed logical // candidate (operator implementation; every sub-DAG NotMaterialized), no // cost; a composed logical candidate Stage 2 cannot implement is listed // under `pruned` with the reason; -// - plan_selection: per-candidate costs, the selected candidate, and -// every other candidate as rejected (`valid: false`) or costlier. Each -// node's `cost` is one computation's price times `multiplier`, the -// times it is computed: once if materialized, else once per query (or -// materialized node) that reads it, so a shared NotMaterialized node is -// charged once per query. When -// selection fails, `selected` is null, with the rejections, or with -// `error` for any other failure. +// - plan_selection: the selected candidate, chosen among all candidates; +// the shown candidates' costs, and every other shown candidate as +// rejected (`valid: false`) or costlier. Each node's `cost` is one +// computation's price times `multiplier`, the times it is computed: +// once if materialized, else once per query (or materialized node) +// that reads it, so a shared NotMaterialized node is charged once per +// query. When selection fails, `selected` is null, with the rejections, +// or with `error` for any other failure. // // `--promql` may repeat. `--epsilon`/`--delta` apply to every `--promql` // query; without them the queries are exact. `--interval-ms` is the source // cadence PromQL needs (default 15000). -use std::rc::Rc; +use std::collections::HashSet; -use asap_aware_mapping::per_computation_candidates::{ - compose_logical_candidate, generate_per_computation_candidates, ComputationCandidate, - PerComputationCandidates, +use asap_aware_mapping::plan_selection::{Rejection, Selection, SelectionError}; +use asap_devtools::planner_layering_examples::{ + declared, example1, pattern_a, pattern_b, PatternARecurrence, }; -use asap_aware_mapping::physical_asap_optimization::optimize_physical_asap; -use asap_aware_mapping::plan_selection::{select_plan, Rejection, Selection, SelectionError}; -use asap_aware_mapping::PlanningModels; +use asap_devtools::planning_stages::{lower_promql_roots, run_planning_stages}; use asap_types::ir::flat::{flatten, FlatDag}; -use asap_types::ir::schema_support::with_promql_series_identity; -use asap_types::ir::{OperatorNode, QueryRoot}; -use asap_types::post_asap::SketchAlgorithm; +use asap_types::ir::QueryRoot; use asap_types::types::AccuracyTarget; use asap_types::workload::{ - AccuracyRequirement, BatchEntry, DataArrival, DataDistribution, DataWorkload, DurationMs, - Evidence, EvidenceSource, LatencyRequirement, PlanningWorkload, Predictability, Query, - QueryLanguage, QueryRecurrence, QueryRequirements, QueryTimeScope, QueryWorkload, Rate, - RepeatedDemand, RepeatingEntry, RepetitionInterval, TimeSelection, + AccuracyRequirement, BatchEntry, DataArrival, DataWorkload, DurationMs, LatencyRequirement, + PlanningWorkload, Query, QueryLanguage, QueryRecurrence, QueryRequirements, QueryWorkload, + RepeatedDemand, }; use serde_json::{json, Value}; -const USAGE: &str = "usage: stage_pipeline (--example planner-layering-1 | --promql ... \ -[--epsilon --delta ] [--interval-ms ]) [--max-candidates ] --out "; +const USAGE: &str = + "usage: stage_pipeline (--example planner-layering-{1,3a,3b} | --promql ... \ +[--epsilon --delta ] [--interval-ms ]) [--max-displayed ] --out "; fn main() { if let Err(message) = run(std::env::args().skip(1).collect()) { @@ -64,7 +64,7 @@ fn run(args: Vec) -> Result<(), String> { let mut example = None; let mut queries = Vec::new(); let (mut epsilon, mut delta, mut interval_ms) = (None, None, 15_000u64); - let mut max_candidates = 64usize; + let mut max_displayed = 64usize; let mut out = None; let mut args = args.into_iter(); while let Some(flag) = args.next() { @@ -76,13 +76,17 @@ fn run(args: Vec) -> Result<(), String> { "--epsilon" => epsilon = Some(number(value()?)?), "--delta" => delta = Some(number(value()?)?), "--interval-ms" => interval_ms = value()?.parse().map_err(|e| format!("{e}"))?, - "--max-candidates" => max_candidates = value()?.parse().map_err(|e| format!("{e}"))?, + "--max-displayed" => max_displayed = value()?.parse().map_err(|e| format!("{e}"))?, "--out" => out = Some(value()?), other => return Err(format!("unknown argument {other}")), } } let workload = match (example.as_deref(), queries.is_empty()) { - (Some("planner-layering-1"), true) => planner_layering_example1(), + (Some("planner-layering-1"), true) => example1(), + (Some("planner-layering-3a"), true) => { + pattern_a(PatternARecurrence::OnceAdHoc, DataArrival::Mixed) + } + (Some("planner-layering-3b"), true) => pattern_b(), (Some(other), true) => return Err(format!("unknown example {other}")), (None, false) => { let accuracy = match (epsilon, delta) { @@ -96,98 +100,85 @@ fn run(args: Vec) -> Result<(), String> { _ => return Err("give exactly one of --example or --promql".into()), }; let out = out.ok_or("--out is required")?; - let document = stage_pipeline(&workload, max_candidates)?; + let document = stage_pipeline(&workload, max_displayed)?; let text = serde_json::to_string_pretty(&document).map_err(|e| e.to_string())? + "\n"; std::fs::write(&out, text).map_err(|e| format!("{out}: {e}")) } -fn stage_pipeline(workload: &PlanningWorkload, max_candidates: usize) -> Result { - // PromQL rows carry each series' full identity as a column: the row - // representation per-series state needs at runtime. - let roots = asap_frontend_promql::lower_promql_query_workload(workload, 0) - .map_err(|e| format!("lowering: {e}"))? - .into_iter() - .map(|root| match root { - QueryRoot::Operator(node) => with_promql_series_identity(&node) - .map(QueryRoot::Operator) - .map_err(|e| format!("series identity: {e}")), - scalar => Ok(scalar), - }) - .collect::, _>>()?; - let frontends_dag = export(&roots)?; - let inventory = generate_per_computation_candidates(roots.into_iter().enumerate().collect()) - .map_err(|e| format!("Pass 1: {e}"))?; - let combinations = inventory - .targets +fn stage_pipeline(workload: &PlanningWorkload, max_displayed: usize) -> Result { + let roots = lower_promql_roots(workload)?; + let run = run_planning_stages(workload, roots)?; + // The display keeps the first `max_displayed` candidates and the + // selected one; plan selection has already ranked all of them. + let selected_logical = run + .stage3_plan_selection + .as_ref() + .ok() + .and_then(|selection| { + run.stage2_physical_asap_optimization + .iter() + .find(|p| p.id == selection.selected) + .map(|p| p.from_logical.as_str()) + }); + let shown: HashSet<&str> = run + .stage1_logical_asap_optimization .iter() - .map(|target| target.choices.candidates.len()) - .product::(); - let owners = target_owners(&inventory); - let mut candidates = Vec::new(); - let mut physical = Vec::new(); - let mut pruned = Vec::new(); - let mut choice = vec![0; inventory.targets.len()]; - for index in 0..combinations.min(max_candidates) { - let (id, label) = ( - format!("L{}", index + 1), - label(&inventory, &owners, &choice), - ); - let mut candidate = json!({ "id": id, "label": label }); - // A choice that composition cannot substitute yet is listed with its - // error rather than ending the run. - match compose_logical_candidate(&inventory, &choice) { - Ok(roots) => { - let roots: Vec<_> = roots.into_iter().map(|(_, root)| root).collect(); - candidate["dag"] = json!(export(&roots)?); - match optimize_physical_asap(&id, &roots) { - Ok(mut physical_candidate) => { - physical_candidate.id = format!("P{}", index + 1); - physical_candidate.label = label; - physical.push(physical_candidate); - } - Err(e) => pruned.push( - json!({ "from_logical": id, "label": label, "reason": e.to_string() }), - ), - } - } - Err(error) => candidate["error"] = json!(error.to_string()), - } - candidates.push(candidate); - // Mixed-radix increment: the last target varies fastest. - for (digit, target) in choice.iter_mut().zip(&inventory.targets).rev() { - *digit += 1; - if *digit < target.choices.candidates.len() { - break; - } - *digit = 0; - } - } - let targets: Vec<_> = workload - .query_workload - .entries() - .map(|entry| Some(entry.requirements.accuracy.target())) + .enumerate() + .filter(|(index, c)| *index < max_displayed || Some(c.id.as_str()) == selected_logical) + .map(|(_, c)| c.id.as_str()) + .collect(); + let shown_physical: HashSet<&str> = run + .stage2_physical_asap_optimization + .iter() + .filter(|p| shown.contains(p.from_logical.as_str())) + .map(|p| p.id.as_str()) + .collect(); + // A choice that composition cannot substitute yet is listed with its + // error rather than ending the run. + let candidates: Vec<_> = run + .stage1_logical_asap_optimization + .iter() + .filter(|c| shown.contains(c.id.as_str())) + .map(|c| match &c.roots { + Ok(roots) => json!({ "id": c.id, "label": c.label, "dag": export(roots) }), + Err(error) => json!({ "id": c.id, "label": c.label, "error": error }), + }) .collect(); - let data = workload.data_workload.clone().unwrap_or_default(); - let selection = select_plan(&physical, &targets, &data, PlanningModels::builtin()); - let physical_json: Vec<_> = physical + let physical: Vec<_> = run + .stage2_physical_asap_optimization .iter() + .filter(|p| shown_physical.contains(p.id.as_str())) .map(|p| json!({ "id": p.id, "from_logical": p.from_logical, "label": p.label, "dag": p.dag })) .collect(); + let pruned: Vec<_> = run + .stage2_pruned + .iter() + .filter(|p| shown.contains(p.from_logical.as_str())) + .map(|p| json!({ "from_logical": p.from_logical, "label": p.label, "reason": p.reason })) + .collect(); Ok(json!({ "format": "asap-stage-pipeline/v1", "workload": { "queries": workload_queries(workload) }, - "language_specific_frontends": { "dag": frontends_dag }, + "language_specific_frontends": { "dag": export(&run.stage0_query_language_frontends) }, "logical_asap_optimization": { - "combinations": combinations, - "capped": combinations > max_candidates, + "combinations": run.stage1_logical_asap_optimization.len(), + "capped": candidates.len() < run.stage1_logical_asap_optimization.len(), "candidates": candidates, }, - "physical_asap_optimization": { "candidates": physical_json, "pruned": pruned }, - "plan_selection": plan_selection_json(&selection), + "physical_asap_optimization": { "candidates": physical, "pruned": pruned }, + "plan_selection": plan_selection_json(&run.stage3_plan_selection, |id| shown_physical.contains(id)), })) } -fn plan_selection_json(selection: &Result) -> Value { +/// Plan selection over every candidate, listing the costs and rejections +/// of the `shown` ones. +fn plan_selection_json( + selection: &Result, + shown: impl Fn(&str) -> bool, +) -> Value { + let rejections_json = |rejected: &[Rejection]| -> Vec { + rejections_json(rejected.iter().filter(|r| shown(&r.id))) + }; let selection = match selection { Ok(selection) => selection, Err(SelectionError::NoValidCandidate(rejected)) => { @@ -200,6 +191,7 @@ fn plan_selection_json(selection: &Result) -> Value { let costs: serde_json::Map<_, _> = selection .costs .iter() + .filter(|(id, _)| shown(id)) .map(|(id, cost)| { let per_node: serde_json::Map<_, _> = cost .per_node @@ -224,92 +216,16 @@ fn plan_selection_json(selection: &Result) -> Value { }) } -fn rejections_json(rejected: &[Rejection]) -> Vec { +fn rejections_json<'a>(rejected: impl Iterator) -> Vec { rejected - .iter() .map(|r| json!({ "id": r.id, "valid": r.valid, "reason": r.reason })) .collect() } /// A logical DAG as a flat node list: node `i` is `nodes[i]`, and children are /// node indices inside each operator. -fn export(roots: &[QueryRoot]) -> Result { - for root in roots { - root.validate_structure() - .map_err(|e| format!("validation: {e}"))?; - } - Ok(flatten(roots).0) -} - -/// The first query whose plan reaches each target (enumeration order). -fn target_owners(inventory: &PerComputationCandidates) -> Vec { - inventory - .targets - .iter() - .map(|target| { - inventory - .roots - .iter() - .position(|(_, root)| { - let operators = match root { - QueryRoot::Operator(node) => vec![node], - QueryRoot::Scalar(expr) => expr.operator_refs(), - }; - operators.into_iter().any(|node| { - OperatorNode::reachable(node) - .iter() - .any(|n| Rc::ptr_eq(n, &target.target)) - }) - }) - .expect("every target is reached from a root") - }) - .collect() -} - -/// E.g. "Q1 exact · Q2 CMS+heap"; exact accumulators are listed in parentheses. -fn label( - inventory: &PerComputationCandidates, - owners: &[usize], - choice: &[usize], -) -> String { - (0..inventory.roots.len()) - .map(|query| { - let mut sketches = Vec::new(); - let mut accumulators = Vec::new(); - for (target, (&owner, &index)) in - inventory.targets.iter().zip(owners.iter().zip(choice)) - { - if owner != query { - continue; - } - match &target.choices.candidates[index] { - ComputationCandidate::Exact => {} - ComputationCandidate::ExactAccumulator { kind, .. } => { - accumulators.push(format!("{kind:?} acc")) - } - ComputationCandidate::AvgAsSumCount => { - accumulators.push("Sum/Count acc".into()) - } - ComputationCandidate::Summary(algorithm) => sketches.push(match algorithm { - SketchAlgorithm::CmsWithHeap => "CMS+heap".to_string(), - SketchAlgorithm::CountSketchWithHeap => "CountSketch+heap".to_string(), - other => format!("{other:?}"), - }), - } - } - let mut text = format!("Q{} ", query + 1); - text += &if sketches.is_empty() { - "exact".to_string() - } else { - sketches.join(" + ") - }; - if !accumulators.is_empty() { - text += &format!(" ({})", accumulators.join(", ")); - } - text - }) - .collect::>() - .join(" · ") +fn export(roots: &[QueryRoot]) -> FlatDag { + flatten(roots).0 } fn workload_queries(workload: &PlanningWorkload) -> Vec { @@ -344,14 +260,6 @@ fn workload_queries(workload: &PlanningWorkload) -> Vec { .collect() } -fn declared(value: T) -> Evidence { - Evidence { - value: Some(value), - source: EvidenceSource::Declared, - ..Default::default() - } -} - fn promql_batch( queries: &[String], accuracy: AccuracyTarget, @@ -385,54 +293,6 @@ fn promql_batch( } } -/// #509 Example 1 (docs/design_docs/proposals/planner-layering.md) over its -/// shared data workload; matches the acceptance spec's test workload. -fn planner_layering_example1() -> PlanningWorkload { - let panel = |query: &str, accuracy, response_latency| RepeatingEntry { - query: Query(query.into()), - demand: RepeatedDemand::FixedInterval(RepetitionInterval(10_000)), - requirements: QueryRequirements { - accuracy: AccuracyRequirement::Explicit(accuracy), - response_latency, - }, - predictability: Predictability::Predictable { known_at: None }, - time_selection: TimeSelection { - scope: QueryTimeScope::RealTime, - lookback: Some(DurationMs(60_000)), - as_of: None, - }, - }; - PlanningWorkload { - query_workload: QueryWorkload { - language: QueryLanguage::PromQL, - query_batch: None, - repeating_queries: Some(vec![ - panel( - "sum by (job) (rate(http_requests_total[1m]))", - AccuracyTarget::Exact, - LatencyRequirement::Unspecified, - ), - panel( - "topk by (job) (10, sum_over_time(http_requests_total[1m]))", - AccuracyTarget::EpsilonDelta { - epsilon: 0.01, - delta: 0.001, - }, - LatencyRequirement::ExplicitMaxMs(100.0), - ), - ]), - }, - data_workload: Some(DataWorkload { - arrival: DataArrival::ContinuouslyIngesting, - data_ingestion_interval: declared(DurationMs(15_000)), - ingestion_volume: Evidence::default(), - ingestion_rate: declared(Rate(1_000_000.0 / 15.0)), - input_cardinality: declared(1_000_000), - distribution: declared(DataDistribution::Zipf), - }), - } -} - #[cfg(test)] mod tests { use super::*; @@ -446,7 +306,8 @@ mod tests { valid: false, reason: "q1: misses target".into(), }]; - let document = plan_selection_json(&Err(SelectionError::NoValidCandidate(rejected))); + let document = + plan_selection_json(&Err(SelectionError::NoValidCandidate(rejected)), |_| true); assert_eq!( document, json!({ @@ -457,14 +318,19 @@ mod tests { ); } - /// A run with no physical candidate (here, none enumerated) still emits - /// every stage, with no selection. + /// With nothing else displayed, the document still shows the selected + /// candidate and its cost, chosen among all candidates. #[test] - fn run_without_candidates_emits_plan_selection() { + fn display_cap_keeps_the_selected_candidate() { let workload = promql_batch(&["sum(m)".into()], AccuracyTarget::Exact, 15_000); let document = stage_pipeline(&workload, 0).unwrap(); - assert_eq!(document["physical_asap_optimization"]["pruned"], json!([])); - assert_eq!(document["plan_selection"]["selected"], Value::Null); + let selected = document["plan_selection"]["selected"].as_str().unwrap(); + let physical = document["physical_asap_optimization"]["candidates"] + .as_array() + .unwrap(); + assert_eq!(physical.len(), 1); + assert_eq!(physical[0]["id"], selected); + assert!(document["plan_selection"]["costs"].get(selected).is_some()); assert_eq!(document["plan_selection"]["rejected"], json!([])); } } diff --git a/crates/devtools/src/lib.rs b/crates/devtools/src/lib.rs index 0316622d..3a44b6ce 100644 --- a/crates/devtools/src/lib.rs +++ b/crates/devtools/src/lib.rs @@ -11,6 +11,13 @@ //! matching front-end crate directly — [`asap_frontend_promql`] (PromQL parser //! only) or [`asap_frontend_sql`] (DataFusion only) — so it never compiles the //! other's parser. +//! +//! [`planning_stages`] and [`planner_layering_examples`] hold the one +//! pipeline and the #509 example workloads that `stage_pipeline` and the +//! planner-layering acceptance tests share. + +pub mod planner_layering_examples; +pub mod planning_stages; pub use asap_frontend_promql::{ lower_promql_workload, lower_promql_workload_with_histograms, PromqlError, diff --git a/crates/devtools/src/planner_layering_examples.rs b/crates/devtools/src/planner_layering_examples.rs new file mode 100644 index 00000000..594693d4 --- /dev/null +++ b/crates/devtools/src/planner_layering_examples.rs @@ -0,0 +1,233 @@ +//! The workloads of #509 Examples 1, 3 and 4 +//! (`docs/design_docs/proposals/planner-layering.md`), shared by +//! `stage_pipeline` and the planner-layering acceptance tests. + +use asap_types::types::AccuracyTarget; +use asap_types::workload::{ + AccuracyRequirement, BatchEntry, DataArrival, DataDistribution, DataWorkload, DurationMs, + Evidence, EvidenceSource, LatencyRequirement, PlanningWorkload, Predictability, Query, + QueryLanguage, QueryRequirements, QueryTimeScope, QueryWorkload, Rate, RepeatedDemand, + RepeatingEntry, RepetitionInterval, TimeSelection, TimestampMs, +}; + +pub fn declared(value: T) -> Evidence { + Evidence { + value: Some(value), + source: EvidenceSource::Declared, + ..Default::default() + } +} + +pub const MINUTE_MS: u64 = 60_000; +/// PromQL's `y` is 365 days. +pub const YEAR_MS: u64 = 365 * 24 * 60 * MINUTE_MS; +/// Pattern A's batch execution time T (2026-01-01T00:00:00Z). +pub const T_MS: u64 = 1_767_225_600_000; + +/// The #509 shared data workload with `arrival`. +pub fn shared_data_workload(arrival: DataArrival) -> DataWorkload { + DataWorkload { + data_ingestion_interval: declared(DurationMs(15_000)), + ingestion_volume: Evidence::default(), + // Data at rest has no ingestion (`validate` rejects a positive rate). + ingestion_rate: match arrival { + DataArrival::AtRest => Evidence::default(), + _ => declared(Rate(1_000_000.0 / 15.0)), + }, + input_cardinality: declared(1_000_000), + distribution: declared(DataDistribution::Zipf), + arrival, + } +} + +pub const EXAMPLE1_Q1: &str = "sum by (job) (rate(http_requests_total[1m]))"; +pub const EXAMPLE1_Q2: &str = "topk by (job) (10, sum_over_time(http_requests_total[1m]))"; + +/// Example 1: two dashboard panels, every 10 s over the last minute. +pub fn example1() -> PlanningWorkload { + let panel = |query: &str, accuracy, response_latency| RepeatingEntry { + query: Query(query.into()), + demand: RepeatedDemand::FixedInterval(RepetitionInterval(10_000)), + requirements: QueryRequirements { + accuracy: AccuracyRequirement::Explicit(accuracy), + response_latency, + }, + predictability: Predictability::Predictable { known_at: None }, + time_selection: TimeSelection { + scope: QueryTimeScope::RealTime, + lookback: Some(DurationMs(MINUTE_MS)), + as_of: None, + }, + }; + PlanningWorkload { + query_workload: QueryWorkload { + language: QueryLanguage::PromQL, + query_batch: None, + repeating_queries: Some(vec![ + panel( + EXAMPLE1_Q1, + AccuracyTarget::Exact, + LatencyRequirement::Unspecified, + ), + panel( + EXAMPLE1_Q2, + AccuracyTarget::EpsilonDelta { + epsilon: 0.01, + delta: 0.001, + }, + LatencyRequirement::ExplicitMaxMs(100.0), + ), + ]), + }, + data_workload: Some(shared_data_workload(DataArrival::ContinuouslyIngesting)), + } +} + +/// Pattern A's five queries: (PromQL, `lookback`, T − `as_of`). +pub const PATTERN_A: [(&str, u64, u64); 5] = [ + ("quantile_over_time(0.99, latency_ms[5y])", 5 * YEAR_MS, 0), + ("quantile_over_time(0.99, latency_ms[1y])", YEAR_MS, 0), + ( + "quantile_over_time(0.99, latency_ms[1y] offset 1y)", + YEAR_MS, + YEAR_MS, + ), + ( + "quantile_over_time(0.99, latency_ms[1y] offset 2y)", + YEAR_MS, + 2 * YEAR_MS, + ), + ( + "quantile_over_time(0.99, latency_ms[3y] offset 2y)", + 3 * YEAR_MS, + 2 * YEAR_MS, + ), +]; + +/// How Pattern A's batch recurs (Example 4 varies it). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PatternARecurrence { + /// As given: one `ad_hoc` batch run once at T. + OnceAdHoc, + /// Example 4: repeated monthly and `Predictable { known_at }`; each run + /// reads the intervals ending at its own evaluation time. + MonthlyPredictable, +} + +fn pattern_a_requirements() -> QueryRequirements { + QueryRequirements { + accuracy: AccuracyRequirement::Explicit(AccuracyTarget::EpsilonDelta { + epsilon: 0.005, + delta: 0.01, + }), + response_latency: LatencyRequirement::Unspecified, + } +} + +/// Example 3, Pattern A: p99 over five historical intervals. +pub fn pattern_a(recurrence: PatternARecurrence, arrival: DataArrival) -> PlanningWorkload { + let (batch, repeating) = match recurrence { + PatternARecurrence::OnceAdHoc => ( + Some( + PATTERN_A + .iter() + .map(|&(query, lookback, before_t)| BatchEntry { + query: Query(query.into()), + requirements: pattern_a_requirements(), + predictability: Predictability::AdHoc, + invocations: 1, + execute_at: Some(TimestampMs(T_MS)), + time_selection: TimeSelection { + scope: QueryTimeScope::Longitudinal, + lookback: Some(DurationMs(lookback)), + as_of: Some(TimestampMs(T_MS - before_t)), + }, + }) + .collect(), + ), + None, + ), + PatternARecurrence::MonthlyPredictable => ( + None, + Some( + PATTERN_A + .iter() + .map(|&(query, lookback, _)| RepeatingEntry { + query: Query(query.into()), + demand: RepeatedDemand::FixedInterval(RepetitionInterval( + (30 * 24 * 60 * MINUTE_MS) as u32, + )), + requirements: pattern_a_requirements(), + predictability: Predictability::Predictable { + known_at: Some(TimestampMs(T_MS)), + }, + time_selection: TimeSelection { + scope: QueryTimeScope::Longitudinal, + lookback: Some(DurationMs(lookback)), + as_of: None, + }, + }) + .collect(), + ), + ), + }; + PlanningWorkload { + query_workload: QueryWorkload { + language: QueryLanguage::PromQL, + query_batch: batch, + repeating_queries: repeating, + }, + data_workload: Some(shared_data_workload(arrival)), + } +} + +pub const PATTERN_B: &str = "quantile_over_time(0.99, latency_ms[5m])"; +pub const PATTERN_B_WINDOW_MS: u64 = 5 * MINUTE_MS; +pub const PATTERN_B_INTERVAL_MS: u64 = MINUTE_MS; + +/// Example 3, Pattern B: a p99 panel over the last 5 min, every minute. +pub fn pattern_b() -> PlanningWorkload { + PlanningWorkload { + query_workload: QueryWorkload { + language: QueryLanguage::PromQL, + query_batch: None, + repeating_queries: Some(vec![RepeatingEntry { + query: Query(PATTERN_B.into()), + demand: RepeatedDemand::FixedInterval(RepetitionInterval( + PATTERN_B_INTERVAL_MS as u32, + )), + requirements: QueryRequirements { + accuracy: AccuracyRequirement::Explicit(AccuracyTarget::EpsilonDelta { + epsilon: 0.01, + delta: 0.01, + }), + response_latency: LatencyRequirement::ExplicitMaxMs(200.0), + }, + predictability: Predictability::Predictable { known_at: None }, + time_selection: TimeSelection { + scope: QueryTimeScope::RealTime, + lookback: Some(DurationMs(PATTERN_B_WINDOW_MS)), + as_of: None, + }, + }]), + }, + data_workload: Some(shared_data_workload(DataArrival::ContinuouslyIngesting)), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Data at rest has no ingestion rate, so the at-rest shared data + /// workload validates. + #[test] + fn at_rest_shared_data_workload_is_valid() { + shared_data_workload(DataArrival::AtRest) + .validate() + .unwrap(); + pattern_a(PatternARecurrence::OnceAdHoc, DataArrival::AtRest) + .validate() + .unwrap(); + } +} diff --git a/crates/devtools/src/planning_stages.rs b/crates/devtools/src/planning_stages.rs new file mode 100644 index 00000000..3a678426 --- /dev/null +++ b/crates/devtools/src/planning_stages.rs @@ -0,0 +1,221 @@ +//! The four planner stages of #509 run end to end over one workload: the +//! one pipeline behind `stage_pipeline` and the planner-layering acceptance +//! tests, so both see the same candidates, ids and selection. + +use std::rc::Rc; + +use asap_aware_mapping::per_computation_candidates::{ + compose_logical_candidate, generate_per_computation_candidates, ComputationCandidate, + PerComputationCandidates, +}; +use asap_aware_mapping::physical_asap_optimization::{optimize_physical_asap, PhysicalCandidate}; +use asap_aware_mapping::plan_selection::{select_plan, Selection, SelectionError}; +use asap_aware_mapping::PlanningModels; +use asap_types::ir::schema_support::with_promql_series_identity; +use asap_types::ir::{OperatorNode, QueryRoot}; +use asap_types::post_asap::SketchAlgorithm; +use asap_types::workload::PlanningWorkload; + +/// One whole-workload logical candidate: a choice of a per-computation +/// candidate for every Pass 1 target. +#[derive(Debug)] +pub struct LogicalCandidate { + /// `L`, numbered in enumeration order from 1. + pub id: String, + /// E.g. "Q1 exact · Q2 CMS+heap". + pub label: String, + /// One root per workload entry, or why composition cannot substitute + /// this choice yet. + pub roots: Result, String>, +} + +/// A composed logical candidate physical ASAP-aware optimization cannot +/// implement. +#[derive(Debug)] +pub struct PrunedCandidate { + pub from_logical: String, + pub label: String, + pub reason: String, +} + +#[derive(Debug)] +pub struct PlanningStagesRun { + /// The query-language frontends' roots, one per workload entry. + pub stage0_query_language_frontends: Vec, + pub stage1_logical_asap_optimization: Vec, + /// `P` implements `L`. + pub stage2_physical_asap_optimization: Vec, + pub stage2_pruned: Vec, + pub stage3_plan_selection: Result, +} + +/// Lower a PromQL workload, each series' identity as a column (the row +/// representation per-series state needs at runtime). +pub fn lower_promql_roots(workload: &PlanningWorkload) -> Result, String> { + asap_frontend_promql::lower_promql_query_workload(workload, 0) + .map_err(|e| format!("lowering: {e}"))? + .into_iter() + .map(|root| match root { + QueryRoot::Operator(node) => with_promql_series_identity(&node) + .map(QueryRoot::Operator) + .map_err(|e| format!("series identity: {e}")), + scalar => Ok(scalar), + }) + .collect() +} + +/// Logical ASAP-aware optimization (Pass 1 and composition; no Pass 2), +/// physical ASAP-aware optimization and plan selection over `roots`, the +/// lowered entries of `workload`. Logical candidates are every choice of a +/// per-computation candidate for every target, the last target varying +/// fastest. Every candidate is enumerated and priced, so the selection is +/// the cheapest of all; a caller that shows fewer cuts only its display. +/// Plan selection reads only the accuracy targets and the data workload. +pub fn run_planning_stages( + workload: &PlanningWorkload, + roots: Vec, +) -> Result { + validate(&roots)?; + let inventory = + generate_per_computation_candidates(roots.iter().cloned().enumerate().collect()) + .map_err(|e| format!("Pass 1: {e}"))?; + let combinations = inventory + .targets + .iter() + .map(|target| target.choices.candidates.len()) + .product::(); + let owners = target_owners(&inventory); + let mut logical = Vec::new(); + let mut physical = Vec::new(); + let mut pruned = Vec::new(); + let mut choice = vec![0; inventory.targets.len()]; + for index in 0..combinations { + let (id, label) = ( + format!("L{}", index + 1), + label(&inventory, &owners, &choice), + ); + let composed = compose_logical_candidate(&inventory, &choice) + .map(|roots| roots.into_iter().map(|(_, root)| root).collect::>()) + .map_err(|e| e.to_string()); + if let Ok(roots) = &composed { + validate(roots)?; + match optimize_physical_asap(&id, roots) { + Ok(mut candidate) => { + candidate.id = format!("P{}", index + 1); + candidate.label = label.clone(); + physical.push(candidate); + } + Err(e) => pruned.push(PrunedCandidate { + from_logical: id.clone(), + label: label.clone(), + reason: e.to_string(), + }), + } + } + logical.push(LogicalCandidate { + id, + label, + roots: composed, + }); + // Mixed-radix increment: the last target varies fastest. + for (digit, target) in choice.iter_mut().zip(&inventory.targets).rev() { + *digit += 1; + if *digit < target.choices.candidates.len() { + break; + } + *digit = 0; + } + } + let targets: Vec<_> = workload + .query_workload + .entries() + .map(|entry| Some(entry.requirements.accuracy.target())) + .collect(); + let data = workload.data_workload.clone().unwrap_or_default(); + let selection = select_plan(&physical, &targets, &data, PlanningModels::builtin()); + Ok(PlanningStagesRun { + stage0_query_language_frontends: roots, + stage1_logical_asap_optimization: logical, + stage2_physical_asap_optimization: physical, + stage2_pruned: pruned, + stage3_plan_selection: selection, + }) +} + +fn validate(roots: &[QueryRoot]) -> Result<(), String> { + roots + .iter() + .try_for_each(|root| root.validate_structure()) + .map_err(|e| format!("validation: {e}")) +} + +/// The first query whose plan reaches each target (enumeration order). +fn target_owners(inventory: &PerComputationCandidates) -> Vec { + inventory + .targets + .iter() + .map(|target| { + inventory + .roots + .iter() + .position(|(_, root)| { + let operators = match root { + QueryRoot::Operator(node) => vec![node], + QueryRoot::Scalar(expr) => expr.operator_refs(), + }; + operators.into_iter().any(|node| { + OperatorNode::reachable(node) + .iter() + .any(|n| Rc::ptr_eq(n, &target.target)) + }) + }) + .expect("every target is reached from a root") + }) + .collect() +} + +/// E.g. "Q1 exact · Q2 CMS+heap"; exact accumulators are listed in parentheses. +fn label( + inventory: &PerComputationCandidates, + owners: &[usize], + choice: &[usize], +) -> String { + (0..inventory.roots.len()) + .map(|query| { + let mut sketches = Vec::new(); + let mut accumulators = Vec::new(); + for (target, (&owner, &index)) in + inventory.targets.iter().zip(owners.iter().zip(choice)) + { + if owner != query { + continue; + } + match &target.choices.candidates[index] { + ComputationCandidate::Exact => {} + ComputationCandidate::ExactAccumulator { kind, .. } => { + accumulators.push(format!("{kind:?} acc")) + } + ComputationCandidate::AvgAsSumCount => { + accumulators.push("Sum/Count acc".into()) + } + ComputationCandidate::Summary(algorithm) => sketches.push(match algorithm { + SketchAlgorithm::CmsWithHeap => "CMS+heap".to_string(), + SketchAlgorithm::CountSketchWithHeap => "CountSketch+heap".to_string(), + other => format!("{other:?}"), + }), + } + } + let mut text = format!("Q{} ", query + 1); + text += &if sketches.is_empty() { + "exact".to_string() + } else { + sketches.join(" + ") + }; + if !accumulators.is_empty() { + text += &format!(" ({})", accumulators.join(", ")); + } + text + }) + .collect::>() + .join(" · ") +} diff --git a/crates/devtools/tests/stage_pipeline.rs b/crates/devtools/tests/stage_pipeline.rs index 897ab193..7fb1f092 100644 --- a/crates/devtools/tests/stage_pipeline.rs +++ b/crates/devtools/tests/stage_pipeline.rs @@ -3,21 +3,30 @@ use std::collections::HashSet; use std::path::PathBuf; use std::process::Command; +use asap_devtools::planner_layering_examples::{pattern_a, PatternARecurrence}; +use asap_devtools::planning_stages::{lower_promql_roots, run_planning_stages}; use asap_types::ir::flat::FlatDag; use asap_types::ir::physical_export::{PhysicalASAPDAG, PhysicalASAPDAGDocument}; use asap_types::ir::QueryRoot; +use asap_types::workload::DataArrival; use serde_json::Value; const COMMITTED: &str = "../../tools/dag-viewer/examples/planner-layering-example1.json"; fn generate(extra: &[&str]) -> Value { + generate_example("planner-layering-1", extra) +} + +fn generate_example(example: &str, extra: &[&str]) -> Value { + // Tests run as threads of one process, so the name needs a per-call counter. + static CALLS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + let call = CALLS.fetch_add(1, std::sync::atomic::Ordering::Relaxed); let out = std::env::temp_dir().join(format!( - "stage_pipeline_{}_{}.json", + "stage_pipeline_{}_{call}_{example}.json", std::process::id(), - extra.len() )); let status = Command::new(env!("CARGO_BIN_EXE_stage_pipeline")) - .args(["--example", "planner-layering-1", "--out"]) + .args(["--example", example, "--out"]) .arg(&out) .args(extra) .status() @@ -122,14 +131,20 @@ fn example1_document_is_valid_and_committed_fixture_is_current() { assert_eq!(accounted, all); } -/// The Cartesian product is cut at `--max-candidates` and says so. +/// `--max-displayed` cuts only what the document shows, says so, and keeps +/// the selection made over all candidates. #[test] -fn candidate_cap_is_recorded() { - let document = generate(&["--max-candidates", "2"]); +fn display_cap_is_recorded_and_keeps_the_selection() { + let full = generate(&[]); + let document = generate(&["--max-displayed", "1"]); let logical = &document["logical_asap_optimization"]; assert_eq!(logical["capped"], true); - assert_eq!(logical["candidates"].as_array().unwrap().len(), 2); - assert!(logical["combinations"].as_u64().unwrap() > 2); + assert!(logical["candidates"].as_array().unwrap().len() < 3); + assert_eq!(logical["combinations"], 3); + assert_eq!( + document["plan_selection"]["selected"], + full["plan_selection"]["selected"] + ); } /// A choice composition cannot substitute is listed with its error, and the @@ -169,3 +184,24 @@ fn uncomposable_candidate_is_recorded_with_its_error() { validated_dag(&candidate["dag"], 1); } } + +/// Example 3a (243 candidates) selects what the planning stages select over +/// every candidate, which the acceptance tests also run: the document's +/// display cap does not change the selection, and the selected plan is +/// displayed with its cost. +#[test] +fn example3a_selects_the_cheapest_of_all_candidates() { + let workload = pattern_a(PatternARecurrence::OnceAdHoc, DataArrival::Mixed); + let run = run_planning_stages(&workload, lower_promql_roots(&workload).unwrap()).unwrap(); + assert_eq!(run.stage1_logical_asap_optimization.len(), 243); + let expected = run.stage3_plan_selection.unwrap().selected; + + let document = generate_example("planner-layering-3a", &[]); + let selection = &document["plan_selection"]; + assert_eq!(selection["selected"], expected.as_str()); + assert!(selection["costs"].get(&expected).is_some()); + let physical = document["physical_asap_optimization"]["candidates"] + .as_array() + .unwrap(); + assert!(physical.iter().any(|p| p["id"] == expected.as_str())); +} diff --git a/crates/integration-tests/Cargo.toml b/crates/integration-tests/Cargo.toml index afa7559b..dcd2a1e0 100644 --- a/crates/integration-tests/Cargo.toml +++ b/crates/integration-tests/Cargo.toml @@ -10,6 +10,7 @@ asap-frontend-sql = { path = "../frontend-sql" } asap-aware-mapping = { path = "../asap-aware-mapping" } [dev-dependencies] +asap-devtools = { path = "../devtools" } asap-planner = { path = "../planner" } asap_sketchlib = { workspace = true } serde_json = "1" diff --git a/crates/integration-tests/tests/planner_layering_common/mod.rs b/crates/integration-tests/tests/planner_layering_common/mod.rs new file mode 100644 index 00000000..913584f0 --- /dev/null +++ b/crates/integration-tests/tests/planner_layering_common/mod.rs @@ -0,0 +1,611 @@ +//! Shared helpers for the #509 Examples 1–4 acceptance tests +//! (`planner_layering_example{1,2,3,4}.rs`). Specs: +//! `docs/design_docs/proposals/planner-layering-example{1,2,3,4}-acceptance.md`. +//! +//! [`run_stages`] runs the real stages through +//! `asap_devtools::planning_stages::run_planning_stages`, the pipeline +//! `stage_pipeline` also runs; the example workloads come from +//! `asap_devtools::planner_layering_examples`. The functions under "Pending adapters" read properties the IR +//! cannot express yet (window summaries, retention). They return today's +//! only possible answer; the implementer of each feature replaces the body +//! with a read of the new IR. Tests that depend on them are `#[ignore]`d +//! with the feature. +#![allow(dead_code)] + +use std::collections::{BTreeMap, BTreeSet, HashSet}; + +use asap_aware_mapping::physical_asap_optimization::PhysicalCandidate; +use asap_aware_mapping::plan_selection::Selection; +use asap_devtools::planning_stages::{lower_promql_roots, run_planning_stages}; +use asap_types::ir::flat::{flatten, FlatDag, NodeId}; +use asap_types::ir::physical_export::PhysicalASAPDAG; +use asap_types::ir::{ASAPOp, Operator, QueryRoot}; +use asap_types::post_asap::{ + FieldDataType, Materialization, SketchAlgorithm, SketchParams, SketchStatistic, +}; +use asap_types::workload::{PlanningWorkload, QueryRecurrence, RepeatedDemand}; + +pub use asap_devtools::planner_layering_examples::*; + +pub type Payload = Operator; + +/// Lower and plan a PromQL workload. +pub fn run_promql(workload: &PlanningWorkload) -> Run { + run_stages(workload, lower_promql(workload)) +} + +/// Whether `form` answers a query window by merging several summaries. +pub fn needs_merge(form: WindowForm, query_window_ms: u64) -> bool { + match form { + WindowForm::None => false, + WindowForm::Sliding { length_ms, .. } => length_ms < query_window_ms, + WindowForm::Tumbling { .. } | WindowForm::ExponentialHistogram { .. } => true, + } +} + +// ── Query-language frontends ────────────────────────────────────────── + +/// PromQL workload roots, each series' identity as a column. +pub fn lower_promql(workload: &PlanningWorkload) -> Vec { + lower_promql_roots(workload).expect("workload lowers") +} + +/// The non-ASAP operator's kind or the aggregate's first measure as +/// `aggregate:`, for each node of one query's frontend DAG, sorted +/// (node ids are post-order; compare as a set of operations). +pub fn frontend_operations(root: &QueryRoot) -> Vec { + let dag = flatten(std::slice::from_ref(root)).0; + let mut ops: Vec = dag.nodes.iter().map(|n| operation(&n.operator)).collect(); + ops.sort(); + ops +} + +/// The aggregate's first measure as `aggregate:`, else the operator's +/// kind in snake case. +pub fn operation(payload: &Payload) -> String { + let json = serde_json::to_value(payload).expect("serializes"); + match json["NonASAP"]["Aggregate"]["measures"][0]["kind"].as_str() { + Some(measure) => format!("aggregate:{measure}"), + None => snake_case(payload.kind_name()), + } +} + +// ── Pipeline ───────────────────────────────────────────────────────────── + +/// One logical candidate. `query_roots` has one root per workload entry, in +/// `QueryWorkload::entries()` order; `roots` are the in-memory roots. +#[derive(Debug, Clone)] +pub struct Logical { + pub id: String, + /// The choice of a per-computation candidate for every Pass 1 target. + pub label: String, + pub dag: FlatDag, + pub query_roots: Vec, + pub roots: Vec, +} + +impl Logical { + /// Whether two queries read a common node. Only Pass 2's + /// identical-expression rule (#588) shares a node across queries. + pub fn shared_input(&self) -> bool { + !cross_query_nodes(&self.dag, &self.query_roots).is_empty() + } +} + +/// One physical candidate, derived from the logical candidate +/// `candidate.from_logical`. +#[derive(Debug, Clone)] +pub struct Physical { + pub id: String, + pub from_logical: String, + pub query_roots: Vec, + pub candidate: PhysicalCandidate, +} + +impl Physical { + pub fn dag(&self) -> &PhysicalASAPDAG { + &self.candidate.dag + } +} + +#[derive(Debug, Clone)] +pub struct Run { + pub stage0_query_language_frontends: Logical, + pub logical: Vec, + pub physical: Vec, + /// Logical candidates physical ASAP-aware optimization cannot implement: + /// (logical id, reason). + pub pruned: Vec<(String, String)>, + pub selection: Selection, +} + +impl Run { + pub fn invalid(&self) -> BTreeMap<&str, &str> { + self.selection + .rejected + .iter() + .filter(|r| !r.valid) + .map(|r| (r.id.as_str(), r.reason.as_str())) + .collect() + } + + pub fn cost(&self, id: &str) -> Option { + self.selection.costs.get(id).map(|c| c.total) + } + + pub fn physical(&self, id: &str) -> &Physical { + self.physical.iter().find(|p| p.id == id).expect("id") + } + + pub fn physical_of<'a>(&'a self, logical: &'a Logical) -> impl Iterator { + self.physical + .iter() + .filter(move |p| p.from_logical == logical.id) + } +} + +fn logical(id: String, label: String, roots: Vec) -> Logical { + let dag = flatten(&roots).0; + let query_roots = dag + .roots + .iter() + .map(|root| match root { + QueryRoot::Operator(id) => *id, + QueryRoot::Scalar(_) => panic!("the examples have operator roots"), + }) + .collect(); + Logical { + id, + label, + dag, + query_roots, + roots, + } +} + +/// Every stage over `roots`, the lowered entries of `workload`, with every +/// logical candidate. Every candidate of the examples composes and +/// plan selection selects one. Recurrence and predictability (which +/// materialization needs, Example 4) are not passed to plan selection. +pub fn run_stages(workload: &PlanningWorkload, roots: Vec) -> Run { + let run = run_planning_stages(workload, roots).expect("planning stages run"); + let logical_candidates = run + .stage1_logical_asap_optimization + .into_iter() + .map(|c| { + let roots = c + .roots + .unwrap_or_else(|e| panic!("{} ({}) composes: {e}", c.id, c.label)); + logical(c.id, c.label, roots) + }) + .collect(); + let physical = run + .stage2_physical_asap_optimization + .into_iter() + .map(|candidate| Physical { + id: candidate.id.clone(), + from_logical: candidate.from_logical.clone(), + query_roots: candidate.dag.roots.clone(), + candidate, + }) + .collect(); + Run { + stage0_query_language_frontends: logical( + "stage0-query-language-frontends".into(), + "query-language frontends".into(), + run.stage0_query_language_frontends, + ), + logical: logical_candidates, + physical, + pruned: run + .stage2_pruned + .into_iter() + .map(|p| (p.from_logical, p.reason)) + .collect(), + selection: run.stage3_plan_selection.expect("plan selection selects"), + } +} + +// ── DAG helpers ────────────────────────────────────────────────────────── + +/// The logical (flat) and physical DAGs share node ids and payloads; the +/// helpers below read only those and each node's producers and consumers. +pub trait ExportedDag { + fn producers(&self, consumer: NodeId) -> Vec; + fn consumers(&self, producer: NodeId) -> Vec; + fn node_ids(&self) -> Vec; + fn payload(&self, id: NodeId) -> &Payload; +} + +impl ExportedDag for FlatDag { + fn producers(&self, consumer: NodeId) -> Vec { + self.nodes[consumer] + .operator + .children() + .into_iter() + .copied() + .collect() + } + fn consumers(&self, producer: NodeId) -> Vec { + self.node_ids() + .into_iter() + .filter(|&id| self.producers(id).contains(&producer)) + .collect() + } + fn node_ids(&self) -> Vec { + (0..self.nodes.len()).collect() + } + fn payload(&self, id: NodeId) -> &Payload { + &self.nodes[id].operator + } +} + +impl ExportedDag for PhysicalASAPDAG { + fn producers(&self, consumer: NodeId) -> Vec { + let edges = self.edges.iter().filter(|e| e.consumer == consumer); + edges.map(|e| e.producer).collect() + } + fn consumers(&self, producer: NodeId) -> Vec { + let edges = self.edges.iter().filter(|e| e.producer == producer); + edges.map(|e| e.consumer).collect() + } + fn node_ids(&self) -> Vec { + self.nodes.iter().map(|n| n.id).collect() + } + fn payload(&self, id: NodeId) -> &Payload { + &self + .nodes + .iter() + .find(|n| n.id == id) + .expect("node") + .payload + } +} + +/// Every node `root` depends on, including itself. +pub fn closure(dag: &impl ExportedDag, root: NodeId) -> HashSet { + let mut seen = HashSet::from([root]); + let mut stack = vec![root]; + while let Some(node) = stack.pop() { + for producer in dag.producers(node) { + if seen.insert(producer) { + stack.push(producer); + } + } + } + seen +} + +/// The queries (indexes into `query_roots`) whose result depends on `node`. +pub fn readers(dag: &impl ExportedDag, query_roots: &[NodeId], node: NodeId) -> BTreeSet { + query_roots + .iter() + .enumerate() + .filter(|(_, &root)| closure(dag, root).contains(&node)) + .map(|(q, _)| q) + .collect() +} + +/// The non-ASAP operator's kind in snake case (`"scan"`, `"time_range"`, …). +pub fn relational(payload: &Payload) -> Option { + match payload { + Operator::NonASAP(_) => Some(snake_case(payload.kind_name())), + Operator::ASAP(_) => None, + } +} + +fn snake_case(name: &str) -> String { + let mut out = String::new(); + for (i, c) in name.chars().enumerate() { + if c.is_ascii_uppercase() && i > 0 { + out.push('_'); + } + out.push(c.to_ascii_lowercase()); + } + out +} + +pub fn is_summary(payload: &Payload) -> bool { + matches!( + payload, + Operator::ASAP( + ASAPOp::SummaryAgg { + family: FieldDataType::Sketch(..), + .. + } | ASAPOp::SummaryEstimate { .. } + | ASAPOp::SummaryMerge { .. } + ) + ) +} + +pub fn is_merge(payload: &Payload) -> bool { + matches!(payload, Operator::ASAP(ASAPOp::SummaryMerge { .. })) +} + +/// The sketch build nodes (`SummaryAgg` over a sketch family) of `dag`, +/// with their algorithm and parameters. +pub fn sketch_builds(dag: &impl ExportedDag) -> Vec<(NodeId, SketchAlgorithm, SketchParams)> { + dag.node_ids() + .into_iter() + .filter_map(|id| match dag.payload(id) { + Operator::ASAP(ASAPOp::SummaryAgg { + family: FieldDataType::Sketch(kind, _), + .. + }) => Some((id, kind.algorithm().clone(), kind.params().clone())), + _ => None, + }) + .collect() +} + +/// The estimation nodes that read `build`, directly or through merges. +pub fn estimates_of(dag: &impl ExportedDag, build: NodeId) -> Vec<(NodeId, SketchStatistic)> { + let mut out = Vec::new(); + let mut stack = vec![build]; + let mut seen = HashSet::new(); + while let Some(node) = stack.pop() { + for consumer in dag.consumers(node) { + if !seen.insert(consumer) { + continue; + } + match dag.payload(consumer) { + Operator::ASAP(ASAPOp::SummaryEstimate { query, .. }) => { + out.push((consumer, query.clone())) + } + Operator::ASAP(ASAPOp::SummaryMerge { .. }) => stack.push(consumer), + _ => {} + } + } + } + out +} + +/// One query's per-computation choice: the sorted sketch algorithms built in +/// its closure, or `"exact"` when it builds none. +pub fn query_option(dag: &impl ExportedDag, query_roots: &[NodeId], query: usize) -> String { + let reach = closure(dag, query_roots[query]); + let mut names: Vec<_> = sketch_builds(dag) + .into_iter() + .filter(|(id, ..)| reach.contains(id)) + .map(|(_, algorithm, _)| format!("{algorithm:?}")) + .collect(); + names.sort(); + if names.is_empty() { + "exact".into() + } else { + names.join("+") + } +} + +/// Nodes reachable from more than one query root. +pub fn cross_query_nodes(dag: &impl ExportedDag, query_roots: &[NodeId]) -> BTreeSet { + dag.node_ids() + .into_iter() + .filter(|&id| readers(dag, query_roots, id).len() > 1) + .collect() +} + +pub fn assert_valid_and_uniquely_named(run: &Run) { + let ids: BTreeSet<_> = run.logical.iter().map(|c| c.id.as_str()).collect(); + assert_eq!(ids.len(), run.logical.len(), "unique logical ids"); + for c in &run.logical { + for root in &c.roots { + root.validate_structure() + .unwrap_or_else(|e| panic!("{}: {e}", c.id)); + } + } +} + +/// Physical ASAP-aware optimization maps the logical candidates one-to-one +/// onto physical ones, so no candidate is discarded before plan selection. +pub fn assert_one_physical_candidate_per_logical(run: &Run) { + assert!(run.pruned.is_empty(), "pruned: {:?}", run.pruned); + let sources: BTreeSet<_> = run + .physical + .iter() + .map(|p| p.from_logical.as_str()) + .collect(); + let logical: BTreeSet<_> = run.logical.iter().map(|c| c.id.as_str()).collect(); + assert_eq!(sources, logical); + assert_eq!(run.physical.len(), run.logical.len()); + let ids: BTreeSet<_> = run.physical.iter().map(|p| p.id.as_str()).collect(); + assert_eq!(ids.len(), run.physical.len(), "unique physical ids"); +} + +/// One selected id; every other candidate rejected once, with a reason. +pub fn assert_selects_one_and_explains_the_rest(run: &Run) { + let all: BTreeSet<_> = run.physical.iter().map(|p| p.id.clone()).collect(); + let mut accounted: BTreeSet<_> = run + .selection + .rejected + .iter() + .map(|r| r.id.clone()) + .collect(); + assert_eq!( + accounted.len(), + run.selection.rejected.len(), + "rejected once each" + ); + assert!(run.selection.rejected.iter().all(|r| !r.reason.is_empty())); + assert!( + accounted.insert(run.selection.selected.clone()), + "selected is not rejected" + ); + assert_eq!(accounted, all); +} + +/// The selected plan costs no more than any valid candidate. +pub fn assert_selects_cheapest_valid(run: &Run) { + let invalid = run.invalid(); + let best = run + .cost(&run.selection.selected) + .expect("selected is priced"); + for p in run + .physical + .iter() + .filter(|p| !invalid.contains_key(p.id.as_str())) + { + assert!(best <= run.cost(&p.id).unwrap(), "{} is cheaper", p.id); + } +} + +/// Each valid candidate's cost has one entry per DAG node, `total` is their +/// sum, and each node's multiplier is [`computations`]. Invalid candidates +/// are not priced. +pub fn assert_each_node_charged_per_consuming_query(run: &Run) { + let invalid = run.invalid(); + for p in &run.physical { + if invalid.contains_key(p.id.as_str()) { + assert!(run.cost(&p.id).is_none(), "{} is priced", p.id); + continue; + } + let cost = &run.selection.costs[&p.id]; + let nodes: BTreeSet<_> = p.dag().nodes.iter().map(|n| n.id).collect(); + let charged: BTreeSet<_> = cost.per_node.keys().copied().collect(); + assert_eq!(charged, nodes, "{}", p.id); + let sum: f64 = cost.per_node.values().map(|c| c.cost).sum(); + assert!( + (cost.total - sum).abs() <= 1e-9 * sum.abs().max(1.0), + "{}", + p.id + ); + let times = computations(p.dag()); + for (id, c) in &cost.per_node { + assert_eq!(c.multiplier, times[id], "{}: multiplier of {id:?}", p.id); + } + } +} + +/// How many times each node is computed per workload evaluation (#509; the +/// rule of plan selection's private `computations`): 1 for a materialized +/// node; for a `NotMaterialized` one, the number of consumers that reach +/// it. A consumer is a query (operator or scalar root) or a materialized +/// node, whose walk starts at its inputs; a walk stops at materialized +/// nodes and counts each node once. +pub fn computations(dag: &PhysicalASAPDAG) -> BTreeMap { + let materialized: BTreeSet<_> = dag + .nodes + .iter() + .filter(|n| n.materialization != Materialization::NotMaterialized) + .map(|n| n.id) + .collect(); + let starts = dag + .roots + .iter() + .map(|&root| vec![root]) + .chain( + dag.scalar_roots + .iter() + .map(|(_, expr)| expr.operator_refs().into_iter().copied().collect()), + ) + .chain(materialized.iter().map(|&id| dag.producers(id))); + let mut times: BTreeMap<_, _> = dag + .nodes + .iter() + .map(|n| (n.id, u64::from(materialized.contains(&n.id)))) + .collect(); + for start in starts { + let mut seen = HashSet::new(); + let mut stack = start; + while let Some(id) = stack.pop() { + if materialized.contains(&id) || !seen.insert(id) { + continue; + } + *times.get_mut(&id).expect("node") += 1; + stack.extend(dag.producers(id)); + } + } + times +} + +/// The evaluation interval of entry `query`, for a fixed-interval repeating +/// entry. +pub fn evaluation_interval_ms(workload: &PlanningWorkload, query: usize) -> Option { + match &workload.query_workload.entries().nth(query)?.recurrence { + QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(i)) => Some(u64::from(i.0)), + _ => None, + } +} + +pub fn lookback_ms(workload: &PlanningWorkload, query: usize) -> u64 { + let entry = workload.query_workload.entries().nth(query).expect("entry"); + let asap_types::workload::DurationMs(ms) = entry.time_selection.lookback.expect("lookback"); + ms +} + +// ── Materialization ────────────────────────────────────────────────────── + +/// A node's materialization option (#509 "Materialization"), without its +/// storage medium. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum MaterializationOption { + /// Runs as data arrives; output stored before any query asks. + MatAtIngestion, + /// Runs when a query first needs it; output kept for later executions + /// and other queries of the batch. + MatAtQuery, + /// Runs at query time for each execution; output discarded. + NotMaterialized, +} + +/// The materialization option physical ASAP-aware optimization assigned to +/// `node` (#667). +pub fn materialization(p: &Physical, node: NodeId) -> MaterializationOption { + let n = p.dag().nodes.iter().find(|n| n.id == node).expect("node"); + match n.materialization { + Materialization::MatAtIngestion(_) => MaterializationOption::MatAtIngestion, + Materialization::MatAtQuery(_) => MaterializationOption::MatAtQuery, + Materialization::NotMaterialized => MaterializationOption::NotMaterialized, + } +} + +pub fn runs_at_ingestion(p: &Physical, node: NodeId) -> bool { + let n = p.dag().nodes.iter().find(|n| n.id == node).expect("node"); + n.output_state.timing == asap_types::post_asap::ExecutionTiming::IngestionTime +} + +/// Every node upstream of an ingestion-time node also runs at ingestion time. +pub fn assert_ingestion_upstream_is_ingestion(run: &Run) { + for p in &run.physical { + for n in &p.dag().nodes { + if !runs_at_ingestion(p, n.id) { + continue; + } + for up in closure(p.dag(), n.id) { + assert!( + runs_at_ingestion(p, up), + "{}: {up:?} feeds ingestion-time {:?} at query time", + p.id, + n.id + ); + } + } + } +} + +// ── Pending adapters ───────────────────────────────────────────────────── + +/// The window summary a summary build node maintains (#509 Pass 2, +/// window-composition rule). +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum WindowForm { + /// Rebuilt from the query window's raw samples (no window summary). + None, + /// Windows of `length_ms` starting every `slide_ms`. + Sliding { length_ms: u64, slide_ms: u64 }, + /// Back-to-back windows of `length_ms`. + Tumbling { length_ms: u64 }, + /// EH buckets covering `horizon_ms` of history. + ExponentialHistogram { horizon_ms: u64 }, +} + +/// Pending (needs Pass 2 window composition, #601/#628): the IR has no +/// window summary yet, so every build is rebuilt from its query window. +pub fn window_form(_dag: &impl ExportedDag, _build: NodeId) -> WindowForm { + WindowForm::None +} + +/// Pending (needs retention in the physical DAG): how long a materialized +/// node's output is kept, in event time. `None` until the IR records it. +pub fn retention_ms(_p: &Physical, _node: NodeId) -> Option { + None +} diff --git a/crates/integration-tests/tests/planner_layering_example1.rs b/crates/integration-tests/tests/planner_layering_example1.rs new file mode 100644 index 00000000..7f41c365 --- /dev/null +++ b/crates/integration-tests/tests/planner_layering_example1.rs @@ -0,0 +1,625 @@ +//! Acceptance tests for #509 "Example 1: Aggregation over dimensions". +//! +//! Spec: `docs/design_docs/proposals/planner-layering-example1-acceptance.md`. +//! Written by a test designer before the stages existed, then run against +//! the real stages. Tests whose expectation needs a feature not in the stack +//! yet are `#[ignore]`d with the PR or issue that delivers it. +//! +//! Scope: logical ASAP-aware optimization = Pass 1 only (no Pass 2); +//! physical ASAP-aware optimization = operator implementation with every +//! sub-DAG `NotMaterialized`. Counts are 1 → 3 → 3 → 1, a subset of the +//! doc's 1 → 54 → 156 → 1. + +mod planner_layering_common; + +use std::collections::{BTreeMap, BTreeSet, HashSet}; + +use asap_aware_mapping::plan_selection::select_plan; +use asap_aware_mapping::PlanningModels; +use asap_types::ir::flat::NodeId; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator, QueryRoot, ScalarExpr, SortKey}; +use asap_types::post_asap::{FieldDataType, GroupingStrategy, HydraKind, SketchAlgorithm}; +use asap_types::post_asap::{Materialization, Medium}; +use asap_types::types::AccuracyTarget; +use planner_layering_common::*; + +// ── Example 1 workload ─────────────────────────────────────────────────── + +fn pipeline() -> Run { + run_promql(&example1()) +} + +// ── Helpers ────────────────────────────────────────────────────────────── + +/// Q2's per-computation option for its top-k, read off its summary build node. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +enum Q2Option { + Exact, + CountMinHeapPerJob, + CountSketchHeapPerJob, + Hydra, +} + +const PASS1_Q2_OPTIONS: [Q2Option; 3] = [ + Q2Option::Exact, + Q2Option::CountMinHeapPerJob, + Q2Option::CountSketchHeapPerJob, +]; + +/// Sketch families built in `nodes`, as Example 1's Q2 options. +fn sketch_options(dag: &impl ExportedDag, nodes: &HashSet) -> BTreeSet { + nodes + .iter() + .filter_map(|&id| match dag.payload(id) { + Operator::ASAP(ASAPOp::SummaryAgg { + family: FieldDataType::Sketch(kind, grouping), + .. + }) => Some(match (grouping, kind.algorithm()) { + ( + GroupingStrategy::SharedMultiSubpopulation { + kind: HydraKind::HydraCms, + .. + }, + _, + ) => Q2Option::Hydra, + (GroupingStrategy::PerSubpopulationInstance, SketchAlgorithm::CmsWithHeap) => { + Q2Option::CountMinHeapPerJob + } + ( + GroupingStrategy::PerSubpopulationInstance, + SketchAlgorithm::CountSketchWithHeap, + ) => Q2Option::CountSketchHeapPerJob, + (other, algorithm) => { + panic!("summary family outside Example 1: {algorithm:?} {other:?}") + } + }), + _ => None, + }) + .collect() +} + +fn roots(query_roots: &[NodeId]) -> (NodeId, NodeId) { + assert_eq!(query_roots.len(), 2, "every candidate covers Q1 and Q2"); + (query_roots[0], query_roots[1]) +} + +/// Q2's option and whether Q1 and Q2 share any node, for one candidate. +fn classify(dag: &impl ExportedDag, query_roots: &[NodeId]) -> (Q2Option, bool) { + let (q1, q2) = roots(query_roots); + let (c1, c2) = (closure(dag, q1), closure(dag, q2)); + let options = sketch_options(dag, &c2); + assert!(options.len() <= 1, "Q2 uses one option: {options:?}"); + let option = options.into_iter().next().unwrap_or(Q2Option::Exact); + (option, !c1.is_disjoint(&c2)) +} + +/// The candidate with Q2 `option` whose queries share no node (#588 adds a +/// shared-input variant of each). +fn physical_with(run: &Run, option: Q2Option) -> &Physical { + let mut found = run + .physical + .iter() + .filter(|p| classify(p.dag(), &p.query_roots) == (option, false)); + let p = found.next().unwrap_or_else(|| panic!("no {option:?}")); + assert!(found.next().is_none(), "one {option:?} candidate"); + p +} + +// ── Workload ───────────────────────────────────────────────────────────── + +/// The encoded workload is valid and normalizes to Q1 then Q2. +#[test] +fn workload_encodes_example1() { + let workload = example1(); + workload.validate().expect("valid workload"); + let entries: Vec<_> = workload.query_workload.entries().collect(); + assert_eq!(entries.len(), 2); + assert_eq!(entries[0].query.0, EXAMPLE1_Q1); + assert_eq!(entries[1].query.0, EXAMPLE1_Q2); +} + +// ── Query-language frontends ────────────────────────────────────────── + +/// The PromQL frontend lowers each query to the doc's chain. +#[test] +fn stage0_query_language_frontends_lower_each_query_to_doc_chain() { + let roots = lower_promql(&example1()); + let chains: Vec<_> = roots.iter().map(frontend_operations).collect(); + assert_eq!( + chains, + [ + ["aggregate:rate", "aggregate:sum", "scan", "time_range"], + ["aggregate:sum", "aggregate:top_k", "scan", "time_range"], + ] + ); +} + +/// The frontends yield one workload DAG with one root per query and no summaries. +#[test] +fn stage0_query_language_frontends_one_summary_free_workload_dag() { + let stage0_query_language_frontends = pipeline().stage0_query_language_frontends; + roots(&stage0_query_language_frontends.query_roots); + assert!(stage0_query_language_frontends + .dag + .nodes + .iter() + .all(|n| !is_summary(&n.operator))); +} + +/// The frontends keep Q1 and Q2 separate; sharing is a logical ASAP-aware optimization decision. +#[test] +fn stage0_query_language_frontends_queries_do_not_share_nodes() { + let stage0_query_language_frontends = pipeline().stage0_query_language_frontends; + assert!(!stage0_query_language_frontends.shared_input()); +} + +// ── Logical ASAP-aware optimization ────────────────────────────────────── + +/// Pass 1 gives 1 × 3 = 3 candidates without a shared input: Q2's top-k is +/// exact, Count-Min + heap or CountSketch + heap. (#588 adds a shared-input +/// variant of each.) +#[test] +fn stage1_logical_asap_optimization_has_three_pass1_candidates() { + let run = pipeline(); + let found: Vec<_> = run + .logical + .iter() + .map(|c| classify(&c.dag, &c.query_roots)) + .filter(|(_, shared)| !shared) + .collect(); + let expected: Vec<_> = PASS1_Q2_OPTIONS.into_iter().map(|o| (o, false)).collect(); + let mut sorted = found.clone(); + sorted.sort(); + assert_eq!(sorted, expected, "each Q2 option exactly once: {found:?}"); +} + +/// Q1 is exact in every candidate: no summary is reachable from its root. +#[test] +fn stage1_logical_asap_optimization_q1_is_always_exact() { + let run = pipeline(); + for c in &run.logical { + let (q1, _) = roots(&c.query_roots); + assert!( + closure(&c.dag, q1) + .iter() + .all(|&id| !is_summary(c.dag.payload(id))), + "{}: Q1 reaches a summary", + c.id + ); + } +} + +/// Every Pass 1 target's exact option is its exact accumulator: Q1's rate and sum and Q2's sum_over_time. +#[test] +fn stage1_logical_asap_optimization_exact_targets_use_accumulators() { + let run = pipeline(); + for c in &run.logical { + let accumulators = c + .dag + .nodes + .iter() + .filter(|n| { + matches!( + n.operator, + Operator::ASAP(ASAPOp::FinalizeExactAccumulator { .. }) + ) + }) + .count(); + assert_eq!(accumulators, 3, "{}", c.id); + } +} + +/// Q2's summary families are Count-Min + heap and CountSketch + heap, one per job. +#[test] +fn stage1_logical_asap_optimization_q2_summary_families() { + let run = pipeline(); + let families: BTreeSet<_> = run + .logical + .iter() + .flat_map(|c| { + let (_, q2) = roots(&c.query_roots); + sketch_options(&c.dag, &closure(&c.dag, q2)) + }) + .collect(); + assert_eq!( + families, + BTreeSet::from([ + Q2Option::CountMinHeapPerJob, + Q2Option::CountSketchHeapPerJob + ]) + ); +} + +/// Pass 1 also offers Hydra (HydraCms over all (job, series) keys, a heap per job) for Q2's top-k. +#[test] +#[ignore = "needs Pass 1 Hydra for grouped approximate counts (#605)"] +fn stage1_logical_asap_optimization_q2_offers_hydra() { + let run = pipeline(); + assert!(run + .logical + .iter() + .any(|c| classify(&c.dag, &c.query_roots).0 == Q2Option::Hydra)); +} + +/// Pass 2's identical-expression rule adds a shared-input variant of every Q2 option and keeps the independent one. +#[test] +#[ignore = "needs the Pass 2 identical-expression rule (#588)"] +fn stage1_logical_asap_optimization_keeps_independent_and_shared_variants() { + let run = pipeline(); + let found: Vec<_> = run + .logical + .iter() + .map(|c| classify(&c.dag, &c.query_roots)) + .collect(); + for option in PASS1_Q2_OPTIONS { + assert!(found.contains(&(option, false)), "{option:?} independent"); + assert!(found.contains(&(option, true)), "{option:?} shared"); + } +} + +/// Only the raw input is ever shared between Q1 and Q2; no summary is shared. +#[test] +fn stage1_logical_asap_optimization_shares_input_but_never_a_summary() { + let run = pipeline(); + for c in &run.logical { + for id in cross_query_nodes(&c.dag, &c.query_roots) { + let p = c.dag.payload(id); + assert!( + matches!(relational(p).as_deref(), Some("scan" | "time_range")), + "{}: shared node {id:?} is not the range selector input: {p:?}", + c.id + ); + } + } +} + +/// Logical candidates are valid DAGs with unique ids. +#[test] +fn stage1_logical_asap_optimization_candidates_are_valid_and_uniquely_named() { + assert_valid_and_uniquely_named(&pipeline()); +} + +// ── Physical ASAP-aware optimization ───────────────────────────────────── + +/// No candidate is discarded before plan selection: the logical candidates +/// map one-to-one, 3 of them without a shared input. +#[test] +fn stage2_physical_asap_optimization_keeps_every_logical_candidate() { + let run = pipeline(); + assert_one_physical_candidate_per_logical(&run); + let unshared = run + .physical + .iter() + .filter(|p| !classify(p.dag(), &p.query_roots).1) + .count(); + assert_eq!(unshared, 3); +} + +/// Each physical candidate keeps its logical Q2 option and input sharing. +#[test] +fn stage2_physical_asap_optimization_preserves_logical_choices() { + let run = pipeline(); + for p in &run.physical { + let source = run.logical.iter().find(|c| c.id == p.from_logical).unwrap(); + assert_eq!( + classify(p.dag(), &p.query_roots), + classify(&source.dag, &source.query_roots), + "{} vs {}", + p.id, + source.id + ); + } +} + +/// Exact TopK is implemented as a sort followed by a limit. +#[test] +fn stage2_physical_asap_optimization_exact_topk_is_sort_then_limit() { + let run = pipeline(); + let p = physical_with(&run, Q2Option::Exact); + let sort_then_limit = p.dag().edges.iter().any(|e| { + relational(p.dag().payload(e.producer)).as_deref() == Some("sort") + && relational(p.dag().payload(e.consumer)).as_deref() == Some("limit") + }); + assert!(sort_then_limit, "{}: no sort → limit", p.id); +} + +/// Exact Q2 sorts each job's series by the ranked value, descending, and a +/// `Project` over the limit restores the logical `[job, topk_10]` columns. +#[test] +fn stage2_physical_asap_optimization_exact_topk_returns_job_and_topk_10() { + let run = pipeline(); + let p = physical_with(&run, Q2Option::Exact); + let QueryRoot::Operator(root) = &p.candidate.roots[1] else { + panic!("{}: Q2 is an operator root", p.id) + }; + let names: Vec<_> = root.schema.fields.iter().map(|f| f.name.as_str()).collect(); + assert_eq!(names, ["job", "topk_10"], "{}", p.id); + let NonASAPOp::Project { child: limit, .. } = root.expect_non_asap() else { + panic!("{}: Q2's root is a Project", p.id) + }; + let NonASAPOp::Limit { + n: Some(10), + child: sort, + .. + } = limit.expect_non_asap() + else { + panic!("{}: a limit 10 under the Project", p.id) + }; + let NonASAPOp::Sort { + keys, + partition_by, + child, + } = sort.expect_non_asap() + else { + panic!("{}: a sort under the limit", p.id) + }; + let [SortKey { + expr: ScalarExpr::Column(ranked), + ascending: false, + .. + }] = keys.as_slice() + else { + panic!("{}: one descending column key: {keys:?}", p.id) + }; + assert_eq!(child.schema.fields[*ranked].name, "value", "{}", p.id); + assert_eq!(partition_by.len(), 1, "{}: partitioned by job", p.id); +} + +/// A summary Q2 is a build node feeding a top-10 estimation node, with no merge. +#[test] +fn stage2_physical_asap_optimization_summary_topk_is_build_then_estimate() { + let run = pipeline(); + for p in &run.physical { + if classify(p.dag(), &p.query_roots).0 == Q2Option::Exact { + continue; + } + assert!( + !p.dag().nodes.iter().any(|n| is_merge(&n.payload)), + "{}: no window summaries, so no merge", + p.id + ); + let build_to_estimate = p.dag().edges.iter().any(|e| { + matches!( + p.dag().payload(e.producer), + Operator::ASAP(ASAPOp::SummaryAgg { + family: FieldDataType::Sketch(..), + .. + }) + ) && matches!( + p.dag().payload(e.consumer), + Operator::ASAP(ASAPOp::SummaryEstimate { .. }) + ) + }); + assert!(build_to_estimate, "{}: no build → estimate", p.id); + } +} + +/// Every sub-DAG is NotMaterialized, so every node runs at query time. +#[test] +fn stage2_physical_asap_optimization_nothing_is_materialized() { + let run = pipeline(); + for p in &run.physical { + for n in &p.dag().nodes { + assert_eq!( + materialization(p, n.id), + MaterializationOption::NotMaterialized, + "{}: {:?}", + p.id, + n.id + ); + assert!(!runs_at_ingestion(p, n.id), "{}: {:?}", p.id, n.id); + } + } +} + +/// Compile `p` in the physical planner (the runtime capability check), with +/// the samples each time range reads as its input. +fn compile_in_runtime(p: &Physical) -> Result<(), String> { + use asap_physical_operators::physical_planner::{compile, InputContract}; + use std::sync::Arc; + let dag = p.dag(); + let mut inputs = BTreeMap::new(); + let mut pending = dag.roots.clone(); + let mut seen = HashSet::new(); + while let Some(id) = pending.pop() { + if !seen.insert(id) { + continue; + } + if relational(dag.payload(id)).as_deref() == Some("time_range") { + let node = dag.nodes.iter().find(|n| n.id == id).unwrap(); + let schema = Arc::new(node.output_schema.clone()); + inputs.insert(id as u64, InputContract::bounded(schema)); + } else { + pending.extend(dag.producers(id)); + } + } + let roots: Vec = dag.roots.iter().map(|&id| id as u64).collect(); + compile(dag, inputs, &roots) + .map(|_| ()) + .map_err(|e| format!("{} ({}): {e}", p.id, p.candidate.label)) +} + +/// Runtime capability check (added by the implementer, not part of the +/// spec): the physical planner compiles every physical candidate. +#[test] +#[ignore = "runtime gaps: CMS+heap needs a non-negative weight contract (#593); \ + CountSketch+heap's SummaryEstimate{TopK} output is not the keyed-evaluation \ + shape the physical planner builds (#579)"] +fn stage2_physical_asap_optimization_every_candidate_compiles_in_the_physical_planner() { + let run = pipeline(); + let failures: Vec<_> = run + .physical + .iter() + .filter_map(|p| compile_in_runtime(p).err()) + .collect(); + assert!(failures.is_empty(), "{failures:#?}"); +} + +// ── Plan selection ─────────────────────────────────────────────────────── + +/// Plan selection selects one candidate and gives every other one a reason. +#[test] +fn stage3_plan_selection_selects_one_and_explains_the_rest() { + assert_selects_one_and_explains_the_rest(&pipeline()); +} + +/// The exact Q2 is selected. Count-Min + heap is invalid because its weights +/// are not proven non-negative; CountSketch + heap is invalid because its +/// top-k output has no guarantee and the deployment has no evidence for it. +#[test] +fn stage3_plan_selection_selects_exact_q2_and_rejects_both_heaps() { + let run = pipeline(); + assert_eq!( + run.selection.selected, + physical_with(&run, Q2Option::Exact).id + ); + let invalid = run.invalid(); + // Only heap candidates are invalid (with or without a shared input). + for id in invalid.keys() { + let p = run.physical(id); + assert_ne!( + classify(p.dag(), &p.query_roots).0, + Q2Option::Exact, + "{invalid:?}" + ); + } + let cms = &physical_with(&run, Q2Option::CountMinHeapPerJob).id; + assert!( + invalid[cms.as_str()].contains("non-negative"), + "{invalid:?}" + ); + let count_sketch = &physical_with(&run, Q2Option::CountSketchHeapPerJob).id; + assert!( + invalid[count_sketch.as_str()].contains("no guarantee"), + "{invalid:?}" + ); +} + +/// The selected plan is the cheapest valid candidate for the whole workload. +#[test] +fn stage3_plan_selection_selects_cheapest_valid() { + assert_selects_cheapest_valid(&pipeline()); +} + +/// Every node of a valid candidate is charged once per query that reads it: nothing is materialized, so a shared node is recomputed for each. +#[test] +fn stage3_plan_selection_charges_each_node_per_consuming_query() { + assert_each_node_charged_per_consuming_query(&pipeline()); +} + +/// A hand-built candidate with a materialized node: Q1's plan read by two +/// queries, its `time_range` kept `MatAtQuery`. Plan selection charges the +/// two accumulators (rate, sum) and their finalizations twice, once per query; the walks from +/// both roots stop at the materialized `time_range`, which is computed once +/// and reads the scan once. +fn materialized_case() -> Run { + let mut workload = example1(); + let repeating = workload.query_workload.repeating_queries.as_mut().unwrap(); + repeating.truncate(1); + let run = run_promql(&workload); + let mut p = run.physical[0].clone(); + p.candidate.roots = vec![p.candidate.roots[0].clone(); 2]; + let dag = &mut p.candidate.dag; + dag.roots = vec![dag.roots[0]; 2]; + p.query_roots = dag.roots.clone(); + let time_range = dag + .nodes + .iter_mut() + .find(|n| relational(&n.payload).as_deref() == Some("time_range")) + .unwrap(); + time_range.materialization = Materialization::MatAtQuery(Medium::Memory); + let selection = select_plan( + std::slice::from_ref(&p.candidate), + &[Some(AccuracyTarget::Exact), Some(AccuracyTarget::Exact)], + workload.data_workload.as_ref().unwrap(), + PlanningModels::builtin(), + ) + .unwrap(); + Run { + physical: vec![p], + selection, + ..run + } +} + +/// The charging helper agrees with plan selection when a node is +/// materialized. +#[test] +fn stage3_plan_selection_charging_helper_follows_plan_selection_with_a_materialized_node() { + let run = materialized_case(); + let p = &run.physical[0]; + let mut by_kind: Vec<_> = run.selection.costs[&p.id] + .per_node + .iter() + .map(|(id, c)| (operation(p.dag().payload(*id)), c.multiplier)) + .collect(); + by_kind.sort(); + let expected = [ + ("finalize_exact_accumulator", 2), + ("finalize_exact_accumulator", 2), + ("scan", 1), + ("summary_agg", 2), + ("summary_agg", 2), + ("time_range", 1), + ]; + assert_eq!( + by_kind, + expected.map(|(kind, times)| (kind.to_string(), times)) + ); + assert_each_node_charged_per_consuming_query(&run); +} + +/// Charging the scan under a materialized node once per query is caught. +#[test] +#[should_panic(expected = "multiplier of")] +fn stage3_plan_selection_charging_helper_rejects_recounting_below_a_materialized_node() { + let mut run = materialized_case(); + let p = run.physical[0].clone(); + let scan = p + .dag() + .nodes + .iter() + .find(|n| relational(&n.payload).as_deref() == Some("scan")) + .unwrap() + .id; + run.selection + .costs + .get_mut(&p.id) + .unwrap() + .per_node + .get_mut(&scan) + .unwrap() + .multiplier = 2; + assert_each_node_charged_per_consuming_query(&run); +} + +/// The plan selection picks compiles in the physical planner (added by the +/// implementer, not part of the spec). +#[test] +fn stage3_plan_selection_selected_plan_compiles_in_the_physical_planner() { + let run = pipeline(); + compile_in_runtime(run.physical(&run.selection.selected)).unwrap(); +} + +/// Sharing the input never costs more than reading it separately. +#[test] +#[ignore = "needs the Pass 2 identical-expression rule (#588): there are no shared-input variants to compare"] +fn stage3_plan_selection_shared_input_is_not_costlier() { + let run = pipeline(); + let by_combo: BTreeMap<_, _> = run + .physical + .iter() + .filter_map(|p| Some((classify(p.dag(), &p.query_roots), run.cost(&p.id)?))) + .collect(); + let mut compared = 0; + for option in PASS1_Q2_OPTIONS { + if let (Some(shared), Some(separate)) = ( + by_combo.get(&(option, true)), + by_combo.get(&(option, false)), + ) { + assert!(shared <= separate, "{option:?}"); + compared += 1; + } + } + assert!(compared > 0); +} diff --git a/crates/integration-tests/tests/planner_layering_example2.rs b/crates/integration-tests/tests/planner_layering_example2.rs new file mode 100644 index 00000000..78e013be --- /dev/null +++ b/crates/integration-tests/tests/planner_layering_example2.rs @@ -0,0 +1,562 @@ +//! Acceptance tests for #509 "Example 2: One summary for several +//! computations — the summary-capability rule in Pass 2". +//! +//! Spec: `docs/design_docs/proposals/planner-layering-example2-acceptance.md`. +//! Written by a test designer who does not implement the stages. Tests that +//! need an unimplemented feature are `#[ignore]`d, naming it and the PR that +//! delivers it. +//! +//! The doc's workload is SQL. The SQL frontend lowers Q1 to a distinct count +//! but not Q2 and Q3 to `Entropy` and `L2` (the doc's own TODO, #598), so +//! the planning stages run on a PromQL stand-in with the same structure: +//! three statistics of one input over the same 1-min window, through the +//! frontend's `distinct_over_time`, `entropy_over_time` and `l2_over_time`. + +mod planner_layering_common; + +use std::collections::BTreeSet; + +use asap_frontend_sql::{lower_sql_dialect, SqlCatalog}; +use asap_types::ir::QueryRoot; +use asap_types::post_asap::{Field, Schema, SketchAlgorithm, SketchParams, SketchStatistic}; +use asap_types::pre_asap::DataType; +use asap_types::types::AccuracyTarget; +use asap_types::workload::{ + AccuracyRequirement, DataArrival, DataDistribution, DataWorkload, DurationMs, + LatencyRequirement, PlanningWorkload, Predictability, Query, QueryLanguage, QueryRequirements, + QueryTimeScope, QueryWorkload, Rate, RepeatedDemand, RepeatingEntry, RepetitionInterval, + SqlDialect, TimeSelection, +}; +use planner_layering_common::*; + +// ── Example 2 workload ─────────────────────────────────────────────────── + +const SQL_Q1: &str = + "SELECT COUNT(DISTINCT src_ip) FROM flows WHERE ts >= now() - INTERVAL '1 minute'"; +const SQL_Q2: &str = "SELECT -SUM(p * LN(p)) FROM (\ + SELECT COUNT(*) * 1.0 / SUM(COUNT(*)) OVER () AS p FROM flows \ + WHERE ts >= now() - INTERVAL '1 minute' GROUP BY src_ip)"; +const SQL_Q3: &str = "SELECT SQRT(SUM(c * c)) FROM (\ + SELECT src_ip, COUNT(*) AS c FROM flows \ + WHERE ts >= now() - INTERVAL '1 minute' GROUP BY src_ip)"; + +/// PromQL stand-in for Q1–Q3 (see the module docs). +const PROMQL_Q1: &str = "distinct_over_time(flows_src_ip[1m])"; +const PROMQL_Q2: &str = "entropy_over_time(flows_src_ip[1m])"; +const PROMQL_Q3: &str = "l2_over_time(flows_src_ip[1m])"; + +/// (ε, δ) of Q1, Q2, Q3. Q3 is the strictest. +const ACCURACY: [(f64, f64); 3] = [(0.02, 0.01), (0.05, 0.01), (0.01, 0.01)]; + +fn panel(query: &str, (epsilon, delta): (f64, f64)) -> RepeatingEntry { + RepeatingEntry { + query: Query(query.into()), + demand: RepeatedDemand::FixedInterval(RepetitionInterval(10_000)), + requirements: QueryRequirements { + accuracy: AccuracyRequirement::Explicit(AccuracyTarget::EpsilonDelta { + epsilon, + delta, + }), + response_latency: LatencyRequirement::Unspecified, + }, + predictability: Predictability::Predictable { known_at: None }, + time_selection: TimeSelection { + scope: QueryTimeScope::RealTime, + lookback: Some(DurationMs(60_000)), + as_of: None, + }, + } +} + +/// The shared data workload with 10,000,000 distinct source IPs. +fn data_workload(ingestion_interval: bool) -> DataWorkload { + DataWorkload { + arrival: DataArrival::ContinuouslyIngesting, + data_ingestion_interval: if ingestion_interval { + declared(DurationMs(15_000)) + } else { + Default::default() + }, + ingestion_volume: Default::default(), + ingestion_rate: declared(Rate(1_000_000.0 / 15.0)), + input_cardinality: declared(10_000_000), + distribution: declared(DataDistribution::Zipf), + } +} + +fn workload(language: QueryLanguage, queries: [&str; 3]) -> PlanningWorkload { + let sql = matches!(language, QueryLanguage::SQL(_)); + PlanningWorkload { + query_workload: QueryWorkload { + language, + query_batch: None, + repeating_queries: Some( + queries + .iter() + .zip(ACCURACY) + .map(|(q, accuracy)| panel(q, accuracy)) + .collect(), + ), + }, + // `data_ingestion_interval` is not needed for SQL; PromQL needs it. + data_workload: Some(data_workload(!sql)), + } +} + +/// Example 2 as the doc writes it. +fn sql_workload() -> PlanningWorkload { + workload( + QueryLanguage::SQL(SqlDialect::DataFusionSQL), + [SQL_Q1, SQL_Q2, SQL_Q3], + ) +} + +fn standin_workload() -> PlanningWorkload { + workload(QueryLanguage::PromQL, [PROMQL_Q1, PROMQL_Q2, PROMQL_Q3]) +} + +/// `flows(ts, src_ip)`: the doc names only these two columns. +fn catalog() -> SqlCatalog { + SqlCatalog::new().with_table( + "flows", + Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("src_ip", DataType::Utf8, false), + ], + 0, + vec![], + ), + ) +} + +async fn lower_sql(workload: &PlanningWorkload) -> Vec { + let mut roots = Vec::new(); + for entry in workload.query_workload.entries() { + let root = lower_sql_dialect( + &entry.query.0, + &catalog(), + SqlDialect::DataFusionSQL, + entry.requirements.accuracy.target(), + ) + .await + .unwrap_or_else(|e| panic!("{}: {e}", entry.query.0)); + roots.push(QueryRoot::Operator(root)); + } + roots +} + +fn pipeline() -> Run { + let workload = standin_workload(); + run_stages(&workload, lower_promql(&workload)) +} + +// ── Helpers ────────────────────────────────────────────────────────────── + +/// UnivMon build nodes of `c` with the queries that read each. +fn univmons(c: &Logical) -> Vec<(BTreeSet, SketchParams, BTreeSet)> { + sketch_builds(&c.dag) + .into_iter() + .filter(|(_, algorithm, _)| *algorithm == SketchAlgorithm::UnivMon) + .map(|(id, _, params)| { + let statistics = estimates_of(&c.dag, id) + .into_iter() + .map(|(_, s)| format!("{s:?}")) + .collect(); + (readers(&c.dag, &c.query_roots, id), params, statistics) + }) + .collect() +} + +/// The UnivMon read by every query in `queries` and by no other. +fn shared_univmon(c: &Logical, queries: &[usize]) -> Option { + let want: BTreeSet<_> = queries.iter().copied().collect(); + univmons(c) + .into_iter() + .find(|(readers, ..)| *readers == want) + .map(|(_, params, _)| params) +} + +/// Whether some summary build is read by more than one query. +fn shares_a_summary(c: &Logical) -> bool { + sketch_builds(&c.dag) + .iter() + .any(|(id, ..)| readers(&c.dag, &c.query_roots, *id).len() > 1) +} + +fn options(run: &Run, query: usize) -> BTreeSet { + run.logical + .iter() + .filter(|c| !shares_a_summary(c)) + .map(|c| query_option(&c.dag, &c.query_roots, query)) + .collect() +} + +/// UnivMon state size: heap entries plus counters over every layer. +fn footprint(params: &SketchParams) -> u64 { + let SketchParams::UnivMon { + heap_size, + sketch_rows, + sketch_cols, + layers, + } = params + else { + panic!("UnivMon params: {params:?}"); + }; + u64::from(*heap_size) + u64::from(*sketch_rows) * u64::from(*sketch_cols) * u64::from(*layers) +} + +/// Q`query`'s own UnivMon parameters, from an independent candidate. +fn own_univmon(run: &Run, query: usize) -> SketchParams { + run.logical + .iter() + .filter(|c| !shares_a_summary(c)) + .find_map(|c| shared_univmon(c, &[query])) + .expect("Pass 1 offers UnivMon") +} + +// ── Workload and query-language frontends ───────────────────────────────────────────────── + +/// The encoded SQL workload is valid and normalizes to Q1, Q2, Q3. +#[test] +fn workload_encodes_example2() { + let workload = sql_workload(); + workload.validate().expect("valid workload"); + let queries: Vec<_> = workload + .query_workload + .entries() + .map(|e| e.query.0) + .collect(); + assert_eq!(queries, [SQL_Q1, SQL_Q2, SQL_Q3]); + standin_workload().validate().expect("valid stand-in"); +} + +/// The SQL frontend lowers Q1 to a distinct count over `flows`. +#[tokio::test] +async fn stage0_query_language_frontends_sql_q1_lowers_to_a_distinct_count() { + let roots = lower_sql(&sql_workload()).await; + let ops = frontend_operations(&roots[0]); + assert!(ops.contains(&"scan".to_string()), "{ops:?}"); + assert!( + ops.contains(&"aggregate:cardinality".to_string()), + "{ops:?}" + ); +} + +/// The SQL frontend recognizes Q2 as `Entropy(src_ip)` and Q3 as `L2(src_ip)`. +#[tokio::test] +#[ignore = "needs SQL frontend recognition of the Entropy and L2 idioms (#598)"] +async fn stage0_query_language_frontends_sql_q2_q3_lower_to_entropy_and_l2() { + let roots = lower_sql(&sql_workload()).await; + assert!(frontend_operations(&roots[1]).contains(&"aggregate:frequency_entropy".to_string())); + assert!(frontend_operations(&roots[2]).contains(&"aggregate:frequency_l2".to_string())); +} + +/// Once the SQL forms are recognized, the SQL workload has the stand-in's Pass 1 options. +#[tokio::test] +#[ignore = "needs SQL frontend recognition of the Entropy and L2 idioms (#598)"] +async fn stage1_logical_asap_optimization_sql_has_the_standin_options() { + let workload = sql_workload(); + let sql = run_stages(&workload, lower_sql(&workload).await); + let standin = pipeline(); + for q in 0..3 { + assert_eq!(options(&sql, q), options(&standin, q), "Q{}", q + 1); + } +} + +/// The stand-in lowers to the three statistics, each over a 1-min range of one input. +#[test] +fn stage0_query_language_frontends_standin_lowers_to_three_statistics_of_one_input() { + let roots = lower_promql(&standin_workload()); + let ops: Vec<_> = roots.iter().map(frontend_operations).collect(); + for (ops, statistic) in ops.iter().zip([ + "aggregate:cardinality", + "aggregate:frequency_entropy", + "aggregate:frequency_l2", + ]) { + let mut want = vec!["scan".to_string(), statistic.into(), "time_range".into()]; + want.sort(); + assert_eq!(ops, &want); + } +} + +/// The frontends give one summary-free workload DAG whose queries share no node. +#[test] +fn stage0_query_language_frontends_one_summary_free_dag_without_sharing() { + let run = pipeline(); + let stage0_query_language_frontends = &run.stage0_query_language_frontends; + assert_eq!(stage0_query_language_frontends.query_roots.len(), 3); + assert!(stage0_query_language_frontends + .dag + .nodes + .iter() + .all(|n| !is_summary(&n.operator))); + assert!(cross_query_nodes( + &stage0_query_language_frontends.dag, + &stage0_query_language_frontends.query_roots + ) + .is_empty()); +} + +// ── Logical ASAP-aware optimization ────────────────────────────────────── + +/// Pass 1 offers an exact and a UnivMon option for each of the three statistics. +#[test] +fn stage1_logical_asap_optimization_pass1_offers_exact_and_univmon_for_each_statistic() { + let run = pipeline(); + for q in 0..3 { + let found = options(&run, q); + assert!(found.contains("exact"), "Q{}: {found:?}", q + 1); + assert!(found.contains("UnivMon"), "Q{}: {found:?}", q + 1); + } +} + +/// Pass 1 offers a specialized distinct-count summary for Q1. +#[test] +fn stage1_logical_asap_optimization_pass1_offers_a_distinct_count_summary() { + let found = options(&pipeline(), 0); + assert!( + found + .iter() + .any(|o| ["Hll", "Theta", "Kmv"].contains(&o.as_str())), + "{found:?}" + ); +} + +/// Pass 1 offers a specialized entropy summary for Q2 and a norm summary for Q3. +#[test] +#[ignore = "needs Pass 1 specialized entropy and L2 summary families (no PR yet)"] +fn stage1_logical_asap_optimization_pass1_offers_specialized_entropy_and_l2_summaries() { + let run = pipeline(); + for q in [1, 2] { + let found = options(&run, q); + assert!( + found.iter().any(|o| o != "exact" && o != "UnivMon"), + "Q{}: {found:?}", + q + 1 + ); + } +} + +/// Every combination of the three queries' own options is kept as an independent candidate. +#[test] +fn stage1_logical_asap_optimization_keeps_every_independent_combination() { + let run = pipeline(); + let found: BTreeSet<_> = run + .logical + .iter() + .filter(|c| !shares_a_summary(c)) + .map(|c| { + (0..3) + .map(|q| query_option(&c.dag, &c.query_roots, q)) + .collect::>() + }) + .collect(); + let (o1, o2, o3) = (options(&run, 0), options(&run, 1), options(&run, 2)); + assert_eq!(found.len(), o1.len() * o2.len() * o3.len()); +} + +/// The summary-capability rule adds one UnivMon build feeding a distinct-count, an entropy and an L2 estimate. +#[test] +#[ignore = "needs the Pass 2 summary-capability rule for frequency moments (#595, #596)"] +fn stage1_logical_asap_optimization_summary_capability_adds_one_univmon_for_all_three() { + let run = pipeline(); + let shared = run.logical.iter().find(|c| { + let ums = univmons(c); + ums.len() == 1 && ums[0].0 == BTreeSet::from([0, 1, 2]) + }); + let c = shared.expect("a candidate with one UnivMon read by Q1, Q2 and Q3"); + let statistics = &univmons(c)[0].2; + for s in [ + SketchStatistic::Cardinality, + SketchStatistic::FrequencyEntropy, + SketchStatistic::FrequencyL2, + ] { + assert!(statistics.contains(&format!("{s:?}")), "{statistics:?}"); + } +} + +/// For each pair, a candidate shares one UnivMon between the two while the third keeps each of its own options. +#[test] +#[ignore = "needs the Pass 2 summary-capability rule for frequency moments (#595, #596)"] +fn stage1_logical_asap_optimization_summary_capability_adds_pairwise_shared_univmons() { + let run = pipeline(); + for (pair, third) in [([0, 1], 2), ([0, 2], 1), ([1, 2], 0)] { + let thirds: BTreeSet<_> = run + .logical + .iter() + .filter(|c| shared_univmon(c, &pair).is_some()) + .map(|c| query_option(&c.dag, &c.query_roots, third)) + .collect(); + assert_eq!(thirds, options(&run, third), "pair {pair:?}"); + } +} + +/// A UnivMon shared by several statistics is sized for the strictest consumer. +#[test] +#[ignore = "needs the Pass 2 summary-capability rule for frequency moments (#595, #596)"] +fn stage1_logical_asap_optimization_shared_univmon_is_sized_for_the_strictest_consumer() { + let run = pipeline(); + let strictest = |queries: &[usize]| { + *queries + .iter() + .min_by(|a, b| ACCURACY[**a].0.total_cmp(&ACCURACY[**b].0)) + .unwrap() + }; + let mut seen = 0; + for queries in [vec![0, 1, 2], vec![0, 1], vec![0, 2], vec![1, 2]] { + for c in &run.logical { + if let Some(params) = shared_univmon(c, &queries) { + seen += 1; + assert_eq!(params, own_univmon(&run, strictest(&queries)), "{}", c.id); + } + } + } + assert!(seen > 0, "no shared UnivMon"); +} + +/// Pass 1 sizes UnivMon for its query's target: ε = 0.01 (Q3) needs a larger state than ε = 0.05 (Q2). +#[test] +#[ignore = "needs UnivMon sizing for an accuracy target (#621); its shape is fixed"] +fn stage1_logical_asap_optimization_univmon_is_sized_per_accuracy_target() { + let run = pipeline(); + assert!(footprint(&own_univmon(&run, 2)) > footprint(&own_univmon(&run, 1))); +} + +/// Only a UnivMon, never a specialized summary, is shared across the statistics. +#[test] +fn stage1_logical_asap_optimization_only_univmon_is_shared_across_statistics() { + let run = pipeline(); + for c in &run.logical { + for (id, algorithm, _) in sketch_builds(&c.dag) { + if readers(&c.dag, &c.query_roots, id).len() > 1 { + assert_eq!(algorithm, SketchAlgorithm::UnivMon, "{}", c.id); + } + } + } +} + +/// Logical candidates are valid DAGs with unique ids. +#[test] +fn stage1_logical_asap_optimization_candidates_are_valid_and_uniquely_named() { + assert_valid_and_uniquely_named(&pipeline()); +} + +// ── Physical ASAP-aware optimization ───────────────────────────────────── + +/// No candidate is discarded before plan selection: logical candidates map one-to-one onto physical ones. +#[test] +fn stage2_physical_asap_optimization_keeps_every_logical_candidate() { + assert_one_physical_candidate_per_logical(&pipeline()); +} + +/// Physical ASAP-aware optimization keeps the shared UnivMon as one build node read by all three queries. +#[test] +#[ignore = "needs the Pass 2 summary-capability rule for frequency moments (#595, #596)"] +fn stage2_physical_asap_optimization_keeps_the_shared_univmon_as_one_build() { + let run = pipeline(); + let mut seen = 0; + for c in run + .logical + .iter() + .filter(|c| shared_univmon(c, &[0, 1, 2]).is_some()) + { + for p in run.physical_of(c) { + seen += 1; + let builds: Vec<_> = sketch_builds(p.dag()) + .into_iter() + .filter(|(_, a, _)| *a == SketchAlgorithm::UnivMon) + .collect(); + assert_eq!(builds.len(), 1, "{}", p.id); + assert_eq!(readers(p.dag(), &p.query_roots, builds[0].0).len(), 3); + } + } + assert!(seen > 0); +} + +// ── Plan selection ─────────────────────────────────────────────────────── + +/// Plan selection selects one candidate and gives every other one a reason. +#[test] +fn stage3_plan_selection_selects_one_and_explains_the_rest() { + assert_selects_one_and_explains_the_rest(&pipeline()); +} + +/// The selected plan is the cheapest valid candidate for the whole workload. +#[test] +fn stage3_plan_selection_selects_cheapest_valid() { + assert_selects_cheapest_valid(&pipeline()); +} + +/// Every node of a valid candidate is charged once per query that reads it: nothing is materialized, so a shared node is recomputed for each. +#[test] +fn stage3_plan_selection_charges_each_node_per_consuming_query() { + assert_each_node_charged_per_consuming_query(&pipeline()); +} + +/// A UnivMon estimate has a guarantee, so plan selection judges it against its target instead of rejecting it for having none. +#[test] +#[ignore = "needs a UnivMon accuracy model: L2 in #621; distinct count and entropy have none yet"] +fn stage3_plan_selection_judges_univmon_with_an_accuracy_model() { + let run = pipeline(); + let invalid = run.invalid(); + for c in &run.logical { + for p in run.physical_of(c) { + let Some(reason) = invalid.get(p.id.as_str()) else { + continue; + }; + for q in 0..3 { + if query_option(&c.dag, &c.query_roots, q) == "UnivMon" { + let unguaranteed = format!("q{}: the output has no guarantee", q + 1); + assert!(!reason.starts_with(&unguaranteed), "{}: {reason}", p.id); + } + } + } + } +} + +/// One materialized UnivMon for all three costs no more than three separate UnivMons. +#[test] +#[ignore = "needs one UnivMon for all three (#595, #596), a UnivMon accuracy model (#621) and materializing it (#604): not materialized, it is charged once per consuming query"] +fn stage3_plan_selection_shared_univmon_costs_no_more_than_three() { + let run = pipeline(); + let cost = |c: &Logical| run.cost(&run.physical_of(c).next().unwrap().id); + let separate: Vec<_> = run + .logical + .iter() + .filter(|c| { + !shares_a_summary(c) + && (0..3).all(|q| query_option(&c.dag, &c.query_roots, q) == "UnivMon") + }) + .map(|c| cost(c).unwrap_or_else(|| panic!("{} is not priced", c.id))) + .collect(); + assert!(!separate.is_empty(), "independent all-UnivMon candidate"); + let mut compared = 0; + for shared in run + .logical + .iter() + .filter(|c| shared_univmon(c, &[0, 1, 2]).is_some()) + { + let shared_cost = cost(shared).unwrap_or_else(|| panic!("{} is not priced", shared.id)); + for separate_cost in &separate { + assert!( + shared_cost <= *separate_cost, + "{shared_cost} vs {separate_cost}" + ); + } + compared += 1; + } + assert!(compared > 0); +} + +/// The selected plan updates at most one UnivMon per flow record: separate UnivMons are dominated by the shared one. +#[test] +fn stage3_plan_selection_selected_plan_has_at_most_one_univmon() { + let run = pipeline(); + let selected = run.physical(&run.selection.selected); + let count = sketch_builds(selected.dag()) + .iter() + .filter(|(_, a, _)| *a == SketchAlgorithm::UnivMon) + .count(); + assert!(count <= 1, "{}: {count} UnivMons", selected.id); +} diff --git a/crates/integration-tests/tests/planner_layering_example3.rs b/crates/integration-tests/tests/planner_layering_example3.rs new file mode 100644 index 00000000..96a75dab --- /dev/null +++ b/crates/integration-tests/tests/planner_layering_example3.rs @@ -0,0 +1,449 @@ +//! Acceptance tests for #509 "Example 3: Aggregation over windows — the +//! window-composition rule in Pass 2". +//! +//! Spec: `docs/design_docs/proposals/planner-layering-example3-acceptance.md`. +//! Written by a test designer who does not implement the stages. Tests that +//! need an unimplemented feature are `#[ignore]`d, naming it and the PR that +//! delivers it. Window summaries are read through +//! `planner_layering_common::window_form`, which the window-composition +//! implementer fills in. + +mod planner_layering_common; + +use std::collections::BTreeSet; + +use asap_types::ir::flat::NodeId; +use asap_types::post_asap::{SketchAlgorithm, SketchParams, SketchStatistic}; +use asap_types::workload::DataArrival; +use planner_layering_common::*; + +fn run_a() -> Run { + run_promql(&pattern_a( + PatternARecurrence::OnceAdHoc, + DataArrival::Mixed, + )) +} + +fn run_b() -> Run { + run_promql(&pattern_b()) +} + +/// The per-computation options of `query` across all candidates. +fn options(run: &Run, query: usize) -> BTreeSet { + run.logical + .iter() + .map(|c| query_option(&c.dag, &c.query_roots, query)) + .collect() +} + +fn op(name: &str) -> String { + name.to_string() +} + +/// The summary builds of `c`: (build, algorithm, window form, readers). +fn windowed_builds(c: &Logical) -> Vec<(NodeId, SketchAlgorithm, WindowForm, BTreeSet)> { + sketch_builds(&c.dag) + .into_iter() + .map(|(id, algorithm, _)| { + let form = window_form(&c.dag, id); + (id, algorithm, form, readers(&c.dag, &c.query_roots, id)) + }) + .collect() +} + +// ── Workloads ──────────────────────────────────────────────────────────── + +/// Pattern A is one ad hoc batch of five queries run once at T; Pattern B one repeating query. +#[test] +fn workload_encodes_example3() { + let a = pattern_a(PatternARecurrence::OnceAdHoc, DataArrival::Mixed); + a.validate().expect("valid Pattern A"); + let queries: Vec<_> = a.query_workload.entries().map(|e| e.query.0).collect(); + assert_eq!(queries, PATTERN_A.map(|(q, ..)| q)); + let b = pattern_b(); + b.validate().expect("valid Pattern B"); + assert_eq!(evaluation_interval_ms(&b, 0), Some(PATTERN_B_INTERVAL_MS)); + assert_eq!(lookback_ms(&b, 0), PATTERN_B_WINDOW_MS); +} + +// ── Pattern A: query-language frontends ─────────────────────────────── + +/// Each Pattern A query lowers to scan → (time shift) → range → quantile. +#[test] +fn stage0_query_language_frontends_a_lowers_each_query_to_its_interval() { + let roots = lower_promql(&pattern_a( + PatternARecurrence::OnceAdHoc, + DataArrival::Mixed, + )); + let ops: Vec<_> = roots.iter().map(frontend_operations).collect(); + let plain = vec![op("aggregate:quantile"), op("scan"), op("time_range")]; + let shifted = vec![ + op("aggregate:quantile"), + op("scan"), + op("time_range"), + op("time_shift"), + ]; + assert_eq!( + ops, + [ + plain.clone(), + plain, + shifted.clone(), + shifted.clone(), + shifted + ] + ); +} + +/// The frontends give one summary-free DAG whose five queries share no node. +#[test] +fn stage0_query_language_frontends_a_one_summary_free_dag_without_sharing() { + let run = run_a(); + let stage0_query_language_frontends = &run.stage0_query_language_frontends; + assert_eq!(stage0_query_language_frontends.query_roots.len(), 5); + assert!(stage0_query_language_frontends + .dag + .nodes + .iter() + .all(|n| !is_summary(&n.operator))); + assert!(cross_query_nodes( + &stage0_query_language_frontends.dag, + &stage0_query_language_frontends.query_roots + ) + .is_empty()); +} + +// ── Pattern A: logical ASAP-aware optimization ─────────────────────────── + +/// Pass 1 offers each query an exact and a KLL candidate over its own interval. +#[test] +fn stage1_logical_asap_optimization_a_pass1_offers_exact_and_kll_per_query() { + let run = run_a(); + for q in 0..5 { + let found = options(&run, q); + assert!( + found.contains("exact") && found.contains("Kll"), + "q{}: {found:?}", + q + 1 + ); + } +} + +/// The five independent KLLs: one per query, each read by its query only, all sized for ε = 0.005. +#[test] +fn stage1_logical_asap_optimization_a_keeps_five_independent_klls() { + let run = run_a(); + let all_kll = run + .logical + .iter() + .find(|c| { + !c.shared_input() && (0..5).all(|q| query_option(&c.dag, &c.query_roots, q) == "Kll") + }) + .expect("independent all-KLL candidate"); + let builds = sketch_builds(&all_kll.dag); + assert_eq!(builds.len(), 5); + let readers: BTreeSet<_> = builds + .iter() + .map(|(id, ..)| readers(&all_kll.dag, &all_kll.query_roots, *id)) + .collect(); + assert_eq!(readers, (0..5).map(|q| BTreeSet::from([q])).collect()); + let sizes: BTreeSet<_> = builds + .iter() + .map(|(_, _, params)| match params { + SketchParams::Kll { k } => *k, + other => panic!("{other:?}"), + }) + .collect(); + assert_eq!(sizes.len(), 1, "one ε, one KLL size: {sizes:?}"); +} + +/// The identical-expression rule shares only raw input (scan, range, shift), never a quantile or summary, and keeps the unshared variant. +#[test] +#[ignore = "needs the Pass 2 identical-expression rule (#588)"] +fn stage1_logical_asap_optimization_a_identical_expression_rule_shares_only_raw_input() { + let run = run_a(); + assert!(run.logical.iter().any(|c| c.shared_input())); + assert!(run.logical.iter().any(|c| !c.shared_input())); + for c in &run.logical { + for id in cross_query_nodes(&c.dag, &c.query_roots) { + let kind = relational(c.dag.payload(id)); + assert!( + matches!(kind.as_deref(), Some("scan" | "time_range" | "time_shift")), + "{}: shared {kind:?}", + c.id + ); + } + } +} + +/// One Exponential Histogram of KLLs over [T − 5y, T] serves all five queries, each through its own merge and p99 estimate. +#[test] +#[ignore = "needs Pass 2 window composition across queries: one window summary for all five (#628, #634)"] +fn stage1_logical_asap_optimization_a_window_composition_adds_one_eh_for_all_five() { + let run = run_a(); + let found = run.logical.iter().find(|c| { + windowed_builds(c).iter().any(|(_, a, form, readers)| { + *a == SketchAlgorithm::Kll + && matches!(form, WindowForm::ExponentialHistogram { horizon_ms } if *horizon_ms >= 5 * YEAR_MS) + && readers.len() == 5 + }) + }); + let c = found.expect("a candidate with one EH of KLLs read by all five queries"); + let (eh, ..) = windowed_builds(c) + .into_iter() + .find(|(_, _, form, _)| matches!(form, WindowForm::ExponentialHistogram { .. })) + .unwrap(); + let estimates = estimates_of(&c.dag, eh); + assert_eq!(estimates.len(), 5, "one estimate per query"); + for (estimate, statistic) in estimates { + assert_eq!(statistic, SketchStatistic::Quantile { q: 0.99 }); + let merged = c + .dag + .producers(estimate) + .into_iter() + .any(|p| is_merge(c.dag.payload(p))); + assert!( + merged, + "{}: estimate {estimate:?} does not read a merge", + c.id + ); + } +} + +/// The shared EH candidate is added next to the independent KLL candidates, not instead of them. +#[test] +#[ignore = "needs Pass 2 window composition across queries: one window summary for all five (#628, #634)"] +fn stage1_logical_asap_optimization_a_keeps_independent_and_shared_window_summaries() { + let run = run_a(); + let shared = run.logical.iter().any(|c| { + windowed_builds(c).iter().any(|(_, _, f, r)| { + matches!(f, WindowForm::ExponentialHistogram { .. }) && r.len() == 5 + }) + }); + let independent = run.logical.iter().any(|c| { + let builds = windowed_builds(c); + builds.len() == 5 + && builds.iter().all(|(_, a, f, r)| { + *a == SketchAlgorithm::Kll && *f == WindowForm::None && r.len() == 1 + }) + }); + assert!(shared && independent); +} + +/// Pass 2 adds a candidate for each way of grouping two queries onto one shared window summary. +#[test] +#[ignore = "needs Pass 2 window composition across queries: partial groupings (#628, #634)"] +fn stage1_logical_asap_optimization_a_window_composition_groups_every_pair() { + let run = run_a(); + let groups: BTreeSet<_> = run + .logical + .iter() + .flat_map(windowed_builds) + .filter(|(_, _, f, r)| *f != WindowForm::None && r.len() > 1) + .map(|(.., r)| r) + .collect(); + for a in 0..5 { + for b in a + 1..5 { + assert!( + groups.contains(&BTreeSet::from([a, b])), + "pair q{} q{}", + a + 1, + b + 1 + ); + } + } +} + +// ── Pattern A: plan selection ──────────────────────────────────────────── + +/// Plan selection selects one Pattern A candidate, explains the rest, and picks the cheapest valid one. +#[test] +fn stage3_plan_selection_a_selects_cheapest_valid() { + let run = run_a(); + assert_selects_one_and_explains_the_rest(&run); + assert_selects_cheapest_valid(&run); +} + +/// Every node of a valid candidate is charged once per query that reads it: nothing is materialized, so a shared node is recomputed for each. +#[test] +fn stage3_plan_selection_a_charges_each_node_per_consuming_query() { + assert_each_node_charged_per_consuming_query(&run_a()); +} + +/// Sharing the scan never costs more than scanning separately, for the same per-computation choices. +#[test] +#[ignore = "needs the Pass 2 identical-expression rule (#588) and plan selection row estimates for time-shifted scans and narrower time ranges (#632)"] +fn stage3_plan_selection_a_shared_scan_is_not_costlier() { + let run = run_a(); + let key = |c: &Logical| { + (0..5) + .map(|q| query_option(&c.dag, &c.query_roots, q)) + .collect::>() + }; + let cost = |c: &Logical| run.cost(&run.physical_of(c).next().unwrap().id); + let mut compared = 0; + for shared in run.logical.iter().filter(|c| c.shared_input()) { + let separate = run + .logical + .iter() + .find(|c| !c.shared_input() && key(c) == key(shared)) + .expect("independent counterpart"); + if let (Some(s), Some(i)) = (cost(shared), cost(separate)) { + assert!(s <= i, "{} {s} vs {} {i}", shared.id, separate.id); + compared += 1; + } + } + assert!(compared > 0); +} + +// ── Pattern B: query-language frontends and Pass 1 ──────────────────── + +/// Pattern B lowers to scan → range 5m → quantile, with no summary. +#[test] +fn stage0_query_language_frontends_b_lowers_to_one_range_quantile() { + let run = run_b(); + let roots = lower_promql(&pattern_b()); + assert_eq!( + frontend_operations(&roots[0]), + [op("aggregate:quantile"), op("scan"), op("time_range")] + ); + assert!(run + .stage0_query_language_frontends + .dag + .nodes + .iter() + .all(|n| !is_summary(&n.operator))); +} + +/// Pass 1 offers an exact and a KLL candidate for the 5-min window. +#[test] +fn stage1_logical_asap_optimization_b_pass1_offers_exact_and_kll() { + let found = options(&run_b(), 0); + assert!( + found.contains("exact") && found.contains("Kll"), + "{found:?}" + ); +} + +// ── Pattern B: window composition ──────────────────────────────────────── + +/// Each Pass 1 option gets three window forms: none, a 5-min sliding window with a 1-min slide, and 1-min tumbling windows. +#[test] +#[ignore = "needs Pass 2 window composition: tumbling windows (#601) and sliding windows (no PR yet)"] +fn stage1_logical_asap_optimization_b_window_composition_adds_sliding_and_tumbling_per_option() { + let run = run_b(); + let summaries: BTreeSet<_> = options(&run, 0) + .into_iter() + .filter(|o| o != "exact") + .collect(); + let found: BTreeSet<_> = run + .logical + .iter() + .flat_map(windowed_builds) + .map(|(_, a, f, _)| (format!("{a:?}"), f)) + .collect(); + let forms = [ + WindowForm::None, + WindowForm::Sliding { + length_ms: PATTERN_B_WINDOW_MS, + slide_ms: PATTERN_B_INTERVAL_MS, + }, + WindowForm::Tumbling { + length_ms: PATTERN_B_INTERVAL_MS, + }, + ]; + for option in &summaries { + for form in forms { + assert!(found.contains(&(option.clone(), form)), "{option} {form:?}"); + } + } +} + +/// Window parameters are legal: L divides W, s divides L and the evaluation interval, a tumbling length divides both. +#[test] +fn stage1_logical_asap_optimization_b_window_parameters_are_legal() { + let (w, every) = (PATTERN_B_WINDOW_MS, PATTERN_B_INTERVAL_MS); + for c in &run_b().logical { + for (_, _, form, _) in windowed_builds(c) { + match form { + WindowForm::Sliding { + length_ms, + slide_ms, + } => { + assert_eq!(w % length_ms, 0, "{}: L | W", c.id); + assert_eq!(length_ms % slide_ms, 0, "{}: s | L", c.id); + assert_eq!(every % slide_ms, 0, "{}: s | interval", c.id); + } + WindowForm::Tumbling { length_ms } => { + assert_eq!(w % length_ms, 0, "{}: length | W", c.id); + assert_eq!(every % length_ms, 0, "{}: length | interval", c.id); + } + WindowForm::ExponentialHistogram { .. } => { + panic!("{}: Pattern B has no EH", c.id) + } + WindowForm::None => {} + } + } + } +} + +/// A sliding window with L = W reads one completed window (no merge); tumbling windows merge W / length of them. +#[test] +#[ignore = "needs Pass 2 window composition: tumbling windows (#601) and sliding windows (no PR yet)"] +fn stage1_logical_asap_optimization_b_merge_only_where_the_window_form_needs_it() { + let run = run_b(); + let mut seen = BTreeSet::new(); + for c in &run.logical { + for (build, _, form, _) in windowed_builds(c) { + let merged = estimates_of(&c.dag, build).iter().any(|(e, _)| { + c.dag + .producers(*e) + .into_iter() + .any(|p| is_merge(c.dag.payload(p))) + }); + assert_eq!( + merged, + needs_merge(form, PATTERN_B_WINDOW_MS), + "{}: {form:?}", + c.id + ); + if form != WindowForm::None { + seen.insert(merged); + } + } + } + assert_eq!( + seen, + BTreeSet::from([false, true]), + "both a sliding and a tumbling candidate" + ); +} + +/// A window form that merges is used only with a summary whose states merge (KLL, DDSketch). +#[test] +fn stage1_logical_asap_optimization_window_merges_use_mergeable_summaries() { + for (run, window) in [(run_a(), 5 * YEAR_MS), (run_b(), PATTERN_B_WINDOW_MS)] { + for c in &run.logical { + for (_, algorithm, form, _) in windowed_builds(c) { + if needs_merge(form, window) { + assert!( + matches!(algorithm, SketchAlgorithm::Kll | SketchAlgorithm::DDSketch), + "{}: {algorithm:?} {form:?}", + c.id + ); + } + } + } + } +} + +// ── Pattern B: plan selection ──────────────────────────────────────────── + +/// Plan selection selects one Pattern B candidate, explains the rest, and picks the cheapest valid one. +#[test] +fn stage3_plan_selection_b_selects_cheapest_valid() { + let run = run_b(); + assert_selects_one_and_explains_the_rest(&run); + assert_selects_cheapest_valid(&run); + assert_each_node_charged_per_consuming_query(&run); +} diff --git a/crates/integration-tests/tests/planner_layering_example4.rs b/crates/integration-tests/tests/planner_layering_example4.rs new file mode 100644 index 00000000..ef0db734 --- /dev/null +++ b/crates/integration-tests/tests/planner_layering_example4.rs @@ -0,0 +1,339 @@ +//! Acceptance tests for #509 "Example 4: Materialization of window summaries +//! in physical planning". +//! +//! Spec: `docs/design_docs/proposals/planner-layering-example4-acceptance.md`. +//! Written by a test designer who does not implement the stages. Tests that +//! need an unimplemented feature are `#[ignore]`d, naming it and the PR that +//! delivers it. Materialization is read from the physical DAG (#667); window +//! summaries and retention are read through the pending adapters in +//! `planner_layering_common`, which the implementers fill in. +//! +//! The workloads are Example 3's, with Pattern A's recurrence and data +//! arrival varied as the doc does. + +mod planner_layering_common; + +use std::collections::{BTreeMap, BTreeSet}; + +use asap_types::ir::flat::NodeId; +use asap_types::post_asap::SketchAlgorithm; +use asap_types::workload::{DataArrival, PlanningWorkload}; +use planner_layering_common::*; + +/// Pattern A as given: one ad hoc batch at T over mixed data. +fn once() -> PlanningWorkload { + pattern_a(PatternARecurrence::OnceAdHoc, DataArrival::Mixed) +} + +fn monthly() -> PlanningWorkload { + pattern_a(PatternARecurrence::MonthlyPredictable, DataArrival::Mixed) +} + +fn at_rest() -> PlanningWorkload { + pattern_a(PatternARecurrence::OnceAdHoc, DataArrival::AtRest) +} + +/// The window-summary build of `c` with form `want` (`Some(form)` matches +/// exactly; `None` matches any EH) read by `readers` queries. +fn window_build(c: &Logical, want: Option, readers_count: usize) -> Option { + sketch_builds(&c.dag) + .into_iter() + .find_map(|(id, algorithm, _)| { + let form = window_form(&c.dag, id); + let matches = match want { + Some(want) => form == want, + None => matches!(form, WindowForm::ExponentialHistogram { .. }), + }; + (algorithm == SketchAlgorithm::Kll + && matches + && readers(&c.dag, &c.query_roots, id).len() == readers_count) + .then_some(id) + }) +} + +/// The physical candidates of the logical candidate with a KLL window +/// summary of `form` read by `readers_count` queries, keyed by the +/// materialization of that build. Several logical candidates may match (for +/// example with and without the shared input); the first is used. +fn options_of( + run: &Run, + form: Option, + readers_count: usize, +) -> BTreeMap { + let logical = run + .logical + .iter() + .find(|c| window_build(c, form, readers_count).is_some()) + .unwrap_or_else(|| { + let form = form.map_or("EH".to_string(), |f| format!("{f:?}")); + panic!("no logical candidate with a {form} KLL window summary") + }); + let build = window_build(logical, form, readers_count).unwrap(); + // Physical ASAP-aware optimization renumbers nodes; find the build again + // in each physical DAG. + run.physical_of(logical) + .map(|p| { + let id = sketch_builds(p.dag()) + .into_iter() + .find(|(id, a, _)| { + *a == SketchAlgorithm::Kll + && window_form(p.dag(), *id) == window_form(&logical.dag, build) + }) + .map(|(id, ..)| id) + .expect("the window summary survives physical ASAP-aware optimization"); + (materialization(p, id), (p.id.clone(), id)) + }) + .collect() +} + +fn eh_options(run: &Run) -> BTreeMap { + options_of(run, None, 5) +} + +fn tumbling() -> Option { + Some(WindowForm::Tumbling { + length_ms: PATTERN_B_INTERVAL_MS, + }) +} + +fn sliding() -> Option { + Some(WindowForm::Sliding { + length_ms: PATTERN_B_WINDOW_MS, + slide_ms: PATTERN_B_INTERVAL_MS, + }) +} + +fn cost(run: &Run, id: &str) -> f64 { + run.cost(id).unwrap_or_else(|| panic!("{id} is not priced")) +} + +use MaterializationOption::{MatAtIngestion, MatAtQuery, NotMaterialized}; + +// ── Constraints over every candidate ───────────────────────────────────── + +/// Every node upstream of an ingestion-time node also runs at ingestion time, in every workload variant. +#[test] +fn stage2_physical_asap_optimization_ingestion_time_upstream_is_ingestion_time() { + for workload in [once(), monthly(), at_rest(), pattern_b()] { + assert_ingestion_upstream_is_ingestion(&run_promql(&workload)); + } +} + +/// Data at rest has no ingestion, so no candidate runs anything at ingestion time. +#[test] +fn stage2_physical_asap_optimization_at_rest_runs_nothing_at_ingestion_time() { + let run = run_promql(&at_rest()); + for p in &run.physical { + for n in &p.dag().nodes { + assert!(!runs_at_ingestion(p, n.id), "{}: {:?}", p.id, n.id); + } + } +} + +/// A materialized output is kept for as long as its consumers read: it covers every reader's lookback and offset. +#[test] +fn stage2_physical_asap_optimization_materialized_output_covers_its_consumers() { + let offsets: Vec = PATTERN_A.iter().map(|(_, _, before_t)| *before_t).collect(); + for (workload, offsets) in [ + (once(), offsets.clone()), + (monthly(), offsets.clone()), + (at_rest(), offsets), + (pattern_b(), vec![0]), + ] { + let run = run_promql(&workload); + for p in &run.physical { + for n in &p.dag().nodes { + if materialization(p, n.id) == NotMaterialized { + continue; + } + let need = readers(p.dag(), &p.query_roots, n.id) + .into_iter() + .map(|q| lookback_ms(&workload, q) + offsets[q]) + .max() + .unwrap_or(0); + let kept = retention_ms(p, n.id).unwrap_or_else(|| { + panic!("{}: {:?} is materialized without a retention", p.id, n.id) + }); + assert!(kept >= need, "{}: {:?} kept {kept} < {need}", p.id, n.id); + } + } + } +} + +// ── Pattern A ──────────────────────────────────────────────────────────── + +/// The shared EH candidate yields A1 (query time, kept), A2 (ingestion time) and A3 (not materialized). +#[test] +#[ignore = "needs Pass 2 window segments shared across queries (#628, #634), materialization enumeration (#604) and MatAtQuery (#625)"] +fn stage2_physical_asap_optimization_a_shared_eh_has_three_materialization_options() { + let found: BTreeSet<_> = eh_options(&run_promql(&once())).into_keys().collect(); + assert_eq!( + found, + BTreeSet::from([MatAtIngestion, MatAtQuery, NotMaterialized]) + ); +} + +/// With data at rest, A2 is not generated; A1 and A3 remain. +#[test] +#[ignore = "needs Pass 2 window segments shared across queries (#628, #634), materialization enumeration (#604) and MatAtQuery (#625)"] +fn stage2_physical_asap_optimization_a_at_rest_drops_the_ingestion_time_option() { + let found: BTreeSet<_> = eh_options(&run_promql(&at_rest())).into_keys().collect(); + assert_eq!(found, BTreeSet::from([MatAtQuery, NotMaterialized])); +} + +/// A materialized shared EH is one build node read by all five queries, charged once. +#[test] +#[ignore = "needs Pass 2 window segments shared across queries (#628, #634), materialization enumeration (#604) and MatAtQuery (#625)"] +fn stage2_physical_asap_optimization_a_materialized_eh_is_built_once_for_all_consumers() { + let run = run_promql(&once()); + for (m, (id, build)) in eh_options(&run) { + if m == NotMaterialized { + continue; + } + let p = run.physical(&id); + let ehs = sketch_builds(p.dag()) + .into_iter() + .filter(|(b, ..)| { + matches!( + window_form(p.dag(), *b), + WindowForm::ExponentialHistogram { .. } + ) + }) + .count(); + assert_eq!(ehs, 1, "{id}"); + assert_eq!(readers(p.dag(), &p.query_roots, build).len(), 5, "{id}"); + assert!( + run.selection.costs[&id].per_node.contains_key(&build), + "{id}" + ); + } +} + +/// A3 rebuilds the EH for each of the five queries (plan selection charges a +/// `NotMaterialized` node once per consuming query), so it costs more than A1, +/// which builds it once. +#[test] +#[ignore = "needs Pass 2 window segments shared across queries (#628, #634), materialization enumeration (#604) and MatAtQuery (#625)"] +fn stage3_plan_selection_a_rebuilding_per_query_costs_more_than_building_once() { + let run = run_promql(&once()); + let options = eh_options(&run); + let (a1, a3) = (&options[&MatAtQuery].0, &options[&NotMaterialized].0); + assert!( + cost(&run, a3) > cost(&run, a1), + "A3 {} vs A1 {}", + cost(&run, a3), + cost(&run, a1) + ); +} + +/// As given (run once, ad hoc), A1 is the cheapest of the three: A2 maintains years of history for one batch. +#[test] +#[ignore = "needs Pass 2 window segments shared across queries (#628, #634), materialization enumeration (#604) and MatAtQuery (#625)"] +fn stage3_plan_selection_a_once_adhoc_prefers_the_query_time_eh() { + let run = run_promql(&once()); + let options = eh_options(&run); + let a1 = cost(&run, &options[&MatAtQuery].0); + for (m, (id, _)) in &options { + assert!(a1 <= cost(&run, id), "A1 {a1} vs {m:?} {}", cost(&run, id)); + } +} + +/// Repeated monthly and predictable, A2's maintenance is shared by many batches, so A2 gains on A1. +#[test] +#[ignore = "needs Pass 2 window segments shared across queries (#628, #634), materialization enumeration (#604) and MatAtQuery (#625)"] +fn stage3_plan_selection_a_monthly_amortizes_ingestion_time_maintenance() { + let ratio = |workload: PlanningWorkload| { + let run = run_promql(&workload); + let options = eh_options(&run); + cost(&run, &options[&MatAtIngestion].0) / cost(&run, &options[&MatAtQuery].0) + }; + let (once, monthly) = (ratio(once()), ratio(monthly())); + assert!(monthly < once, "A2/A1: monthly {monthly} vs once {once}"); +} + +// ── Pattern B ──────────────────────────────────────────────────────────── + +/// The 1-min tumbling KLL candidate yields B1 (ingestion time), B2 (not materialized) and B3 (query time, kept). +#[test] +#[ignore = "needs tumbling windows (#601), materialization enumeration (#604) and MatAtQuery panes (#625)"] +fn stage2_physical_asap_optimization_b_tumbling_kll_has_three_materialization_options() { + let found: BTreeSet<_> = options_of(&run_promql(&pattern_b()), tumbling(), 1) + .into_keys() + .collect(); + assert_eq!( + found, + BTreeSet::from([MatAtIngestion, MatAtQuery, NotMaterialized]) + ); +} + +/// B1 builds the tumbling KLLs at ingestion time and merges and estimates at query time. +#[test] +#[ignore = "needs tumbling windows (#601) and materialization enumeration (#604)"] +fn stage2_physical_asap_optimization_b_b1_builds_at_ingestion_and_merges_at_query_time() { + let run = run_promql(&pattern_b()); + let (id, build) = options_of(&run, tumbling(), 1)[&MatAtIngestion].clone(); + let p = run.physical(&id); + assert!(runs_at_ingestion(p, build)); + let estimates = estimates_of(p.dag(), build); + assert!(!estimates.is_empty()); + for (estimate, _) in estimates { + assert!( + !runs_at_ingestion(p, estimate), + "{id}: estimate at ingestion" + ); + let merge = p + .dag() + .producers(estimate) + .into_iter() + .find(|&m| is_merge(p.dag().payload(m))); + let merge = merge.unwrap_or_else(|| panic!("{id}: no merge before the estimate")); + assert!(!runs_at_ingestion(p, merge), "{id}: merge at ingestion"); + } +} + +/// B3 runs nothing at ingestion time; it keeps the tumbling KLLs it builds at query time. +#[test] +#[ignore = "needs tumbling windows (#601) and MatAtQuery panes (#625)"] +fn stage2_physical_asap_optimization_b_b3_keeps_query_time_windows() { + let run = run_promql(&pattern_b()); + let (id, _) = options_of(&run, tumbling(), 1)[&MatAtQuery].clone(); + let p = run.physical(&id); + assert!( + p.dag().nodes.iter().all(|n| !runs_at_ingestion(p, n.id)), + "{id}" + ); +} + +/// The sliding-window KLL is kept from ingestion time or from query time; not materializing it is the no-window plan. +#[test] +#[ignore = "needs sliding windows (no PR yet) and materialization enumeration (#604)"] +fn stage2_physical_asap_optimization_b_sliding_kll_has_two_materialization_options() { + let found: BTreeSet<_> = options_of(&run_promql(&pattern_b()), sliding(), 1) + .into_keys() + .collect(); + assert_eq!(found, BTreeSet::from([MatAtIngestion, MatAtQuery])); +} + +/// B2 rebuilds all five tumbling KLLs at every evaluation, so it costs at least B1 and B3. +#[test] +#[ignore = "needs tumbling windows (#601), materialization enumeration (#604) and MatAtQuery panes (#625)"] +fn stage3_plan_selection_b_rebuilding_every_window_costs_most() { + let run = run_promql(&pattern_b()); + let options = options_of(&run, tumbling(), 1); + let b2 = cost(&run, &options[&NotMaterialized].0); + for m in [MatAtIngestion, MatAtQuery] { + assert!(cost(&run, &options[&m].0) <= b2, "{m:?} vs B2 {b2}"); + } +} + +/// Repeating over arriving data, the built-in models pick B1 among the tumbling options. +#[test] +#[ignore = "needs tumbling windows (#601), materialization enumeration (#604) and MatAtQuery panes (#625)"] +fn stage3_plan_selection_b_prefers_ingestion_time_tumbling_windows() { + let run = run_promql(&pattern_b()); + let options = options_of(&run, tumbling(), 1); + let b1 = cost(&run, &options[&MatAtIngestion].0); + for (m, (id, _)) in &options { + assert!(b1 <= cost(&run, id), "B1 {b1} vs {m:?} {}", cost(&run, id)); + } +} diff --git a/docs/design_docs/proposals/planner-layering-example1-acceptance.md b/docs/design_docs/proposals/planner-layering-example1-acceptance.md new file mode 100644 index 00000000..d54b47e4 --- /dev/null +++ b/docs/design_docs/proposals/planner-layering-example1-acceptance.md @@ -0,0 +1,189 @@ +# Planner layering, Example 1: acceptance spec (MVP) + +Audience: planner designers and the implementers of the planning stages. +Source: [planner-layering.md](planner-layering.md), "Example 1: Aggregation over +dimensions". Tests: `crates/integration-tests/tests/planner_layering_example1.rs` +(helpers in `tests/planner_layering_common/`). Viewer fixture: +`tools/dag-viewer/examples/planner-layering-example1.json`, written by +`stage_pipeline --example planner-layering-1`. + +This spec was written by a test designer before the stages existed. A test +whose expectation needs a feature that is not in the stack yet is +`#[ignore]`d with the PR or issue that delivers it. + +## Scope + +| Stage | Doc (#509) | MVP (this spec) | +|---|---|---| +| Query-language frontends | 1 workload `LogicalDAG` | same | +| Logical ASAP-aware optimization | Pass 1 (3) × identical-expression rule (2) × window-composition rule (3 × 3) = **54** | Pass 1 only = **3**. Pass 2 is not implemented: the identical-expression rule is #588, window composition #601. | +| Physical ASAP-aware optimization | Materialization options per window form = **156** | Physical operator implementation, every sub-DAG `NotMaterialized` = **3** (the doc's Raw plans). Materialization enumeration is #604. | +| Plan selection | 1 plan | 1 plan | + +Every MVP candidate is one of the doc's candidates: in logical ASAP-aware +optimization, the one with separate inputs and no window summary for either +query; in physical ASAP-aware optimization, the one where both queries are +**Raw** (state rebuilt from the last 1 min of raw samples at every refresh). + +## Workload + +Shared data workload: `continuously_ingesting`, 15 s ingestion interval, +1,000,000 series, about 66,667 samples/s, `zipf`, volume unknown. + +| Query | Repeats | `lookback` | `as_of` | Accuracy | Latency | +|---|---|---|---|---|---| +| Q1 `sum by (job) (rate(http_requests_total[1m]))` | 10 s | 1 m | evaluation time | exact | none | +| Q2 `topk by (job) (10, sum_over_time(http_requests_total[1m]))` | 10 s | 1 m | evaluation time | ε = 0.01, δ = 0.001 | ≤ 100 ms | + +## Query-language frontends: 1 candidate + +| Query | Chain | +|---|---| +| Q1 | scan `http_requests_total` → range 1m → rate (per series) → sum by (job) | +| Q2 | scan `http_requests_total` → range 1m → sum_over_time (per series) → topk by (job) (10) | + +Invariants: + +* Exactly one candidate, with one root per query, in workload entry order. +* No summary node (sketch `SummaryAgg`, `SummaryEstimate`, `SummaryMerge`). +* Q1 and Q2 share no node. Sharing is a logical ASAP-aware optimization + decision. + +## Logical ASAP-aware optimization: 3 candidates + +Pass 1 targets and their per-computation candidates: + +| Query | Target | Candidates | +|---|---|---| +| Q1 | per-series `rate` | exact Rate accumulator | +| Q1 | per-`job` `sum` | exact Sum accumulator | +| Q2 | per-series `sum_over_time` | exact Sum accumulator | +| Q2 | per-`job` top-*k* | exact, Count-Min + heap per job, CountSketch + heap per job | + +| Id | Q1 | Q2 top-*k* | +|---|---|---| +| L1 | exact | exact (sort + limit 10 per job) | +| L2 | exact | Count-Min + top-*k* heap, one per job | +| L3 | exact | CountSketch + top-*k* heap, one per job | + +Ids follow the enumeration order; tests identify candidates by structure. + +Invariants: + +* Exactly 3 candidates, one per Q2 option. No duplicates. +* Every candidate covers both queries. +* Q1 never reaches a summary node. +* Every target's exact option is its exact accumulator (`SummaryAgg` with an + exact family → `FinalizeExactAccumulator`): three per candidate. +* Q2's sketch families are exactly {`CmsWithHeap`, `CountSketchWithHeap`}, + each per subpopulation (job). Exact Q2 has no sketch. +* Only the scan and range nodes may be shared between the queries. No summary + is shared. (Today nothing is shared.) +* Nothing is pruned: the heap sketches are admitted by ε = 0.01, δ = 0.001, + and Q1's exact target admits only the exact option. + +Ignored until their feature lands: + +* Hydra (`HydraCms` over all (job, series) keys, a heap per job) as a third + Q2 summary (#605). +* A shared-input variant of every Q2 option next to the independent one + (Pass 2 identical-expression rule, #588). + +## Physical ASAP-aware optimization: 3 candidates + +| Id | From | Q2 physical operators | Materialization | +|---|---|---|---| +| P1 | L1 | sort (partition by job) → limit 10 | all `NotMaterialized` | +| P2 | L2 | CMS + heap build → top-10 estimate | all `NotMaterialized` | +| P3 | L3 | CountSketch + heap build → top-10 estimate | all `NotMaterialized` | + +Q1 is the same in all three: scan → range → rate accumulator → sum +accumulator, each finalized. + +Invariants: + +* Exactly one physical candidate per logical candidate, and none pruned. + `from_logical` is a bijection onto the logical ids, so no candidate is + dropped before plan selection. +* Each physical candidate keeps its logical Q2 option and input sharing. +* Exact top-*k* is implemented as a sort followed by a limit: each job's + series sorted by the ranked value, descending, then a `Project` that + restores the logical root columns `[job, topk_10]`. +* A summary Q2 is a sketch build node feeding an estimation node. There is no + merge node, because there are no window summaries. +* Every node is `NotMaterialized` and runs at query time. + +Runtime capability check (not part of the doc): every candidate compiles in +the physical planner. Ignored on two runtime gaps: CMS + heap needs a +non-negative weight contract (#593), and the CountSketch + heap estimate's +output is not the keyed-evaluation shape the physical planner builds (#579). + +## Plan selection: 1 plan + +| Id | Outcome | Reason | +|---|---|---| +| P1 | selected | the only valid candidate | +| P2 | invalid | Count-Min needs non-negative update weights, and `sum_over_time` weights are not proven non-negative | +| P3 | invalid | a top-*k* output has no guarantee (a heap's bound is per item frequency, not top-*k* membership), and the deployment has no evidence that it meets the target | + +Invariants: + +* One selected id. Every other candidate is listed once as rejected, with a + reason, and marked invalid or valid but costlier. +* Only valid candidates are priced. Each valid candidate's cost has one entry + per DAG node, and `total` is their sum. A materialized node is charged + once. A `NotMaterialized` node is charged once per consumer that reads it + (its multiplier), where a consumer is a query or a materialized node and + a read stops at materialized nodes. Every node is `NotMaterialized` today; + `stage3_plan_selection_charging_helper_*` check the helper on a hand-built candidate with a + materialized node. +* The selected plan is no more expensive than any valid candidate. +* The selected plan compiles in the physical planner. + +Ignored until Pass 2 (#588): for each Q2 option, the shared-input candidate +costs no more than its separate counterpart. + +When #588's shared-input candidates land, each Q2 option has two +candidates. The tests already count only candidates whose queries share no +node (Pass 1's 3 logical and 3 physical), pick a Q2 option's candidate by +(option, no shared input), and check only that every invalid candidate is a +heap one, not that exactly two are invalid, so they need no change. + +The doc's typical winners need window forms or materialization, and its +"Raw with a shared input" needs Pass 2. Latency bounds are not checked, so +whether an exact Q2 misses 100 ms is not tested. + +## Where the doc and the planner differ + +1. **Q2's top-*k* options.** The doc lists "exact, Count-Min + heap, Hydra". + The planner offers exact, Count-Min + heap and CountSketch + heap; Hydra + comes in #605. The tests follow the planner. +2. **Count-Min + heap is invalid.** The doc expects the accuracy model to + judge it. Plan selection admits Count-Min only over weights proven + non-negative, which `sum_over_time` over an arbitrary metric is not + (counter declarations: #593). +3. **No summary is selectable.** The doc expects a summary for Q2 to win in + most deployments. A composed top-*k* estimate has no guarantee, so without + deployment evidence every heap candidate is invalid and the exact plan is + selected. +4. **Physical ASAP-aware optimization is materialization in the doc's + example.** Example 1 lists only window-form and materialization options. + The MVP reads "physical operator implementation" from the physical + ASAP-aware optimization section ("TopK as a sort followed by a limit"), + which gives one physical candidate per logical candidate. +5. **Where exact top-*k* becomes sort + limit.** The Pass 1 diagram draws + exact Q2 as "sort + limit 10 per job", but the frontend emits + `Aggregate{[TopK]}` and the physical ASAP-aware optimization section calls + sort + limit a physical choice. The tests require sort → limit only after + physical ASAP-aware optimization. +6. **Hydra's inner sketch.** The doc says "Hydra over the whole `job` + column" without naming the inner sketch. The test assumes `HydraCms`. +7. **"Shared summary charged once"** does not apply literally: the doc says + no summary is shared in Example 1. Plan selection charges a + `NotMaterialized` node once per consuming query and a materialized one + once; the tests check that rule. A shared input therefore saves nothing + until it is materialized (#604). +8. **Latency.** The doc says an exact top 10 rebuilt at every refresh "may + miss" 100 ms. Plan selection does not check latency yet. +9. **Accuracy of Count-Min heap merges** matters only with tumbling windows, + which are out of MVP scope. diff --git a/docs/design_docs/proposals/planner-layering-example2-acceptance.md b/docs/design_docs/proposals/planner-layering-example2-acceptance.md new file mode 100644 index 00000000..26cd2b82 --- /dev/null +++ b/docs/design_docs/proposals/planner-layering-example2-acceptance.md @@ -0,0 +1,148 @@ +# Planner layering, Example 2: acceptance spec + +Audience: planner designers and the implementers of the summary-capability +rule, UnivMon sizing and accuracy, and SQL `Entropy`/`L2` recognition. +Source: [planner-layering.md](planner-layering.md), "Example 2: One summary for +several computations". Tests: +`crates/integration-tests/tests/planner_layering_example2.rs` (helpers in +`tests/planner_layering_common/`). + +This spec was written by a test designer who does not implement the stages. +Tests run the stages through `planner_layering_common::run_stages`: Pass 1 +and composition, `optimize_physical_asap` and `select_plan`. A test that +needs a missing feature is `#[ignore]`d with the PR that delivers it. + +## Workload + +Three SQL dashboard panels over `flows`, every 10 s, `lookback` 1 m, `as_of` +evaluation time, no latency requirement. + +| Query | Computation | SQL (abridged) | Accuracy | +|---|---|---|---| +| Q1 | `Distinct(src_ip)` | `SELECT COUNT(DISTINCT src_ip) FROM flows WHERE ts >= now() - INTERVAL '1 minute'` | ε = 0.02, δ = 0.01 | +| Q2 | `Entropy(src_ip)` | `SELECT -SUM(p * LN(p)) FROM (SELECT COUNT(*) * 1.0 / SUM(COUNT(*)) OVER () AS p … GROUP BY src_ip)` | ε = 0.05, δ = 0.01 | +| Q3 | `L2(src_ip)` | `SELECT SQRT(SUM(c * c)) FROM (SELECT src_ip, COUNT(*) AS c … GROUP BY src_ip)` | ε = 0.01, δ = 0.01 | + +Data workload: the shared one (`continuously_ingesting`, about 66,667 +samples/s, `zipf`, volume unknown) with `input_cardinality` = 10,000,000 and no +`data_ingestion_interval`. The catalog is `flows(ts, src_ip)`; the doc names +no other column. + +**Stand-in.** The SQL frontend lowers Q1 to a distinct count but Q2 and Q3 to +exact relational plans (the doc's own TODO; #598). The planning stages +therefore run on a PromQL stand-in with the same structure, three statistics +of one input over the same 1-min window: `distinct_over_time(flows_src_ip[1m])`, +`entropy_over_time(flows_src_ip[1m])` and `l2_over_time(flows_src_ip[1m])`, +with Q1–Q3's targets and a 15-s ingestion interval (PromQL needs one). + +## Expected candidates per stage + +| Stage | Doc | Today (stand-in) | +|---|---|---| +| Query-language frontends | One summary-free DAG: `Distinct`, `Entropy`, `L2` over `flows.src_ip`, last 1 m | Same for the stand-in. SQL: Q1 is `aggregate:cardinality`; Q2 and Q3 are `count` per `src_ip` → window function / projection → `sum` (not recognized). | +| Logical ASAP-aware optimization, Pass 1 | Each computation: exact, a specialized summary, UnivMon. 3 × 3 × 3 = **27** | Q1: exact, HLL, Theta, KMV, UnivMon. Q2, Q3: exact, UnivMon (no specialized entropy or norm summary). 5 × 2 × 2 = **20** | +| Logical ASAP-aware optimization, Pass 2 | Summary-capability rule: + 1 (one UnivMon for all three, sized for ε = 0.01) + 9 (3 pairs share a UnivMon × 3 options of the third). Independent kept. **37**. Window composition left out. | Not implemented: **20**, no candidate shares a node. | +| Physical ASAP-aware optimization | Same options as Example 1: no window form → rebuilt from raw samples each refresh. | One physical candidate per logical candidate, every sub-DAG `NotMaterialized`: **20**. | +| Plan selection | One UnivMon sized for ε = 0.01 vs three separate summaries; the shared one usually wins (one update per flow record instead of three). | HLL is invalid (its bound has no failure probability for the (ε, δ) target) and so is every UnivMon (its estimates have no guarantee). Q1 on Theta or KMV with Q2, Q3 exact is valid but costlier. The all-exact plan is selected. | + +## Invariants and tests + +### Query-language frontends + +| Invariant | Test | Status | +|---|---|---| +| The SQL workload is valid, in entry order | `workload_encodes_example2` | passes | +| SQL Q1 lowers to a distinct count over `flows` | `stage0_query_language_frontends_sql_q1_lowers_to_a_distinct_count` | passes | +| SQL Q2 and Q3 lower to `FrequencyEntropy` and `FrequencyL2` | `stage0_query_language_frontends_sql_q2_q3_lower_to_entropy_and_l2` | ignored: #598 | +| The stand-in lowers to the three statistics over one 1-min range | `stage0_query_language_frontends_standin_lowers_to_three_statistics_of_one_input` | passes | +| One summary-free DAG; the queries share no node | `stage0_query_language_frontends_one_summary_free_dag_without_sharing` | passes | + +### Logical ASAP-aware optimization + +| Invariant | Test | Status | +|---|---|---| +| Once recognized, the SQL workload has the stand-in's Pass 1 options | `stage1_logical_asap_optimization_sql_has_the_standin_options` | ignored: #598 (ambiguity 9) | +| Each statistic has an exact and a UnivMon option | `stage1_logical_asap_optimization_pass1_offers_exact_and_univmon_for_each_statistic` | passes | +| Q1 has a specialized distinct-count summary (HLL, Theta or KMV) | `stage1_logical_asap_optimization_pass1_offers_a_distinct_count_summary` | passes | +| Q2 and Q3 have a specialized entropy and norm summary | `stage1_logical_asap_optimization_pass1_offers_specialized_entropy_and_l2_summaries` | ignored: no PR yet | +| Every combination of the queries' own options is kept, with no shared summary | `stage1_logical_asap_optimization_keeps_every_independent_combination` | passes | +| A candidate has one UnivMon build read by all three queries, feeding a distinct-count, an entropy and an L2 estimate | `stage1_logical_asap_optimization_summary_capability_adds_one_univmon_for_all_three` | ignored: #595, #596 | +| For each pair, a candidate shares one UnivMon between the two while the third takes each of its own options | `stage1_logical_asap_optimization_summary_capability_adds_pairwise_shared_univmons` | ignored: #595, #596 | +| A shared UnivMon has the parameters Pass 1 gives its strictest consumer alone | `stage1_logical_asap_optimization_shared_univmon_is_sized_for_the_strictest_consumer` | ignored: #595, #596 | +| UnivMon is sized per target: Q3's (ε = 0.01) is larger than Q2's (ε = 0.05) | `stage1_logical_asap_optimization_univmon_is_sized_per_accuracy_target` | ignored: #621 | +| Only a UnivMon is ever shared across statistics | `stage1_logical_asap_optimization_only_univmon_is_shared_across_statistics` | passes (vacuously today) | +| Candidates are valid DAGs with unique ids | `stage1_logical_asap_optimization_candidates_are_valid_and_uniquely_named` | passes | + +### Physical ASAP-aware optimization + +| Invariant | Test | Status | +|---|---|---| +| One physical candidate per logical candidate, none pruned | `stage2_physical_asap_optimization_keeps_every_logical_candidate` | passes | +| The shared UnivMon stays one build read by all three queries | `stage2_physical_asap_optimization_keeps_the_shared_univmon_as_one_build` | ignored: #595, #596 | + +Materialization and window variants are out of scope: the doc leaves out the +window-composition variants for this example, and a summary with no window +form is rebuilt at every refresh (Example 1's table). + +### Plan selection + +| Invariant | Test | Status | +|---|---|---| +| One selected id; every other candidate rejected once with a reason | `stage3_plan_selection_selects_one_and_explains_the_rest` | passes | +| The selected plan is the cheapest valid one | `stage3_plan_selection_selects_cheapest_valid` | passes | +| Each node of a valid candidate is charged once per consuming query while `NotMaterialized` (all today), once when materialized | `stage3_plan_selection_charges_each_node_per_consuming_query` | passes | +| A UnivMon estimate has a guarantee, so it is judged against its target instead of rejected for having none | `stage3_plan_selection_judges_univmon_with_an_accuracy_model` | ignored: L2 in #621; distinct count and entropy have no PR | +| One materialized UnivMon for all three costs no more than three separate UnivMons | `stage3_plan_selection_shared_univmon_costs_no_more_than_three` | ignored: #595, #596, #621, #604 | +| Selection direction: the selected plan has at most one UnivMon, because one shared UnivMon dominates several over the same input | `stage3_plan_selection_selected_plan_has_at_most_one_univmon` | passes (vacuously today: no UnivMon is valid) | + +The doc's "the shared candidate usually wins" depends on the cost model, so +it is tested only as the dominance above: one build sized for the strictest +consumer costs no more than three builds that include one of that size. Whether +it also beats the exact plans is the model's call. The dominance needs the +shared UnivMon materialized (#604): plan selection charges a `NotMaterialized` +node once per consuming query, so unmaterialized it costs three builds of the +strictest size. + +## Ambiguities and conflicts with the planner + +1. **SQL recognition.** Q2 and Q3 lower to exact relational plans, so the doc's + workload has no `Entropy` or `L2` intent (#598). Hence the stand-in. The + stand-in's statistics are per series (`PerEntity`), the doc's over the + whole column; with one `flows_src_ip` series they coincide. +2. **Pass 1 options.** The doc gives each computation three options. The + planner gives Q1 five (three specialized distinct-count summaries) and Q2, + Q3 two (no specialized entropy or norm summary). The tests check presence, + not counts. +3. **Candidate count.** The doc's 37 excludes the identical-expression rule's + shared-input variant (Example 1 counts it). Neither Pass 2 rule exists yet + (#588, #595), so today there are 20. Tests do not pin a count. +4. **Independent UnivMons on a shared input.** The doc keeps independent + candidates. Once Pass 2 shares the input, identical UnivMon builds may + merge; the tests require the independent combinations only without a + shared summary. +5. **Sizing.** "Sized for the strictest requirement, ε = 0.01" presumes + UnivMon sizing per ε; UnivMon has a fixed shape today (#621). +6. **Accuracy.** Plan selection accepts an output whose guarantee meets the + target or for which the deployment has evidence. No UnivMon estimate has + a guarantee, so every UnivMon candidate is invalid; #621 certifies L2 + only. HLL is rejected against an (ε, δ) target because its bound has no + failure probability; the doc treats a distinct summary as valid. +7. **Pairwise count.** "The third keeps any of its own 3 options" includes its + own UnivMon, sized for itself. The test requires the third's full Pass 1 + option set. +8. **Data workload.** Only `input_cardinality` and `data_ingestion_interval` + change, so the ingestion rate stays about 66,667/s with 10,000,000 keys. +9. **SQL candidates compose.** On `flows(ts, src_ip)`, every logical + candidate of the SQL workload composes, including the CMS and CountSketch + candidates for Q2's and Q3's `COUNT(*) … GROUP BY src_ip`. Until #598 + recognizes the Entropy and L2 idioms, those per-source counts are what + Pass 1 sees, so the SQL options differ from the stand-in's. + +## Devtool + +No `--example planner-layering-2` yet. It needs: + +* SQL `Entropy` and `L2` recognition (#598), so the demo shows the doc's + workload rather than exact relational plans; +* a SQL lowering path with the `flows` catalog in `stage_pipeline` (devtool + only; it lowers PromQL only today). diff --git a/docs/design_docs/proposals/planner-layering-example3-acceptance.md b/docs/design_docs/proposals/planner-layering-example3-acceptance.md new file mode 100644 index 00000000..73cf4877 --- /dev/null +++ b/docs/design_docs/proposals/planner-layering-example3-acceptance.md @@ -0,0 +1,120 @@ +# Planner layering, Example 3: acceptance spec + +Audience: planner designers and the implementer of Pass 2's +window-composition rule. +Source: [planner-layering.md](planner-layering.md), "Example 3: Aggregation over +windows" and the Pass 2 section. Tests: +`crates/integration-tests/tests/planner_layering_example3.rs` (helpers in +`tests/planner_layering_common/`; the Example 3/4 workloads and the stage +pipeline in `asap_devtools::{planner_layering_examples, planning_stages}`). +Devtool: `stage_pipeline --example planner-layering-3a` and +`planner-layering-3b`. Both run every candidate (243 for 3a) through plan +selection; `--max-displayed` (default 64) limits only how many the document +shows, so 3a selects P122, KLL for all five queries. + +This spec was written by a test designer who does not implement the stages. +Tests run the stages through `planner_layering_common::run_stages`, which +calls `run_planning_stages`, the pipeline `stage_pipeline` runs. A test +that needs a missing feature is `#[ignore]`d with the PR that delivers it. + +## Workload + +Shared data workload, except Pattern A's `arrival` is `mixed`. + +**Pattern A.** One `query_batch`, `invocations: 1` at T, `ad_hoc`, ε = 0.005, +δ = 0.01. Tests use T = 2026-01-01T00:00:00Z and a 365-day year. + +| Query | `lookback` | `as_of` | +|---|---|---| +| q1 `quantile_over_time(0.99, latency_ms[5y])` | 5 y | T | +| q2 `quantile_over_time(0.99, latency_ms[1y])` | 1 y | T | +| q3 `quantile_over_time(0.99, latency_ms[1y] offset 1y)` | 1 y | T − 1 y | +| q4 `quantile_over_time(0.99, latency_ms[1y] offset 2y)` | 1 y | T − 2 y | +| q5 `quantile_over_time(0.99, latency_ms[3y] offset 2y)` | 3 y | T − 2 y | + +**Pattern B.** `quantile_over_time(0.99, latency_ms[5m])`, every 1 min, +`lookback` 5 m, `as_of` evaluation time, ε = 0.01, δ = 0.01, ≤ 200 ms. + +## Expected candidates per stage + +| Stage | Doc | Today | +|---|---|---| +| Query-language frontends | A: five scan → range → quantile chains. B: one. | Same; q3–q5 also have a `time_shift`. | +| Logical ASAP-aware optimization, Pass 1 | A: exact or KLL over its own interval per query, 2⁵ = 32. B: exact, KLL. | A: exact, KLL (k = 547), DDSketch per query, 3⁵ = **243**. B: exact, KLL (k = 269), DDSketch, **3**. | +| Logical ASAP-aware optimization, Pass 2 | A: window composition adds one EH of KLLs over [T − 5 y, T], one merge + p99 estimate per query, plus a candidate for every grouping of two or more queries onto shared window summaries ("hundreds"). B: each Pass 1 option × {none, sliding L = 5 min s = 1 min (5 active windows), 1-min tumbling (merge 5)} = 6. Independent kept. | Not implemented: no shared input, no window summaries. | +| Physical ASAP-aware optimization | See Example 4. | One physical candidate per logical candidate, every sub-DAG `NotMaterialized`. | +| Plan selection | Not stated for Example 3. | Every candidate is valid. A: five independent KLLs (DDSketch ties on cost). B: KLL (DDSketch ties). | + +## Invariants and tests + +### Pattern A + +| Invariant | Test | Status | +|---|---|---| +| Both workloads are valid; Pattern B repeats every 1 min over 5 min | `workload_encodes_example3` | passes | +| Each query lowers to scan → (time shift) → range → quantile | `stage0_query_language_frontends_a_lowers_each_query_to_its_interval` | passes | +| One summary-free frontend DAG; the queries share no node | `stage0_query_language_frontends_a_one_summary_free_dag_without_sharing` | passes | +| Each query has an exact and a KLL option | `stage1_logical_asap_optimization_a_pass1_offers_exact_and_kll_per_query` | passes | +| Five independent KLLs, each read by one query, all the same size | `stage1_logical_asap_optimization_a_keeps_five_independent_klls` | passes | +| The identical-expression rule shares only raw input (scan, range, shift) and keeps the unshared variant | `stage1_logical_asap_optimization_a_identical_expression_rule_shares_only_raw_input` | ignored: #588 | +| One EH of KLLs over ≥ 5 y is read by all five queries; each has its own merge feeding a p99 estimate | `stage1_logical_asap_optimization_a_window_composition_adds_one_eh_for_all_five` | ignored: #628, #634 | +| The EH candidate coexists with the five independent KLLs | `stage1_logical_asap_optimization_a_keeps_independent_and_shared_window_summaries` | ignored: #628, #634 | +| Every pair of queries shares a window summary in some candidate | `stage1_logical_asap_optimization_a_window_composition_groups_every_pair` | ignored: #628, #634 | +| One selected id, the rest explained; the selected plan is the cheapest valid one | `stage3_plan_selection_a_selects_cheapest_valid` | passes | +| Each node of a valid candidate is charged once per consuming query while `NotMaterialized` (all today), once when materialized | `stage3_plan_selection_a_charges_each_node_per_consuming_query` | passes | +| For the same per-computation choices, sharing the scan costs no more than separate scans | `stage3_plan_selection_a_shared_scan_is_not_costlier` | ignored: #588, #632 (see ambiguity 7) | + +### Pattern B + +| Invariant | Test | Status | +|---|---|---| +| The query lowers to scan → range 5m → quantile, no summary | `stage0_query_language_frontends_b_lowers_to_one_range_quantile` | passes | +| Exact and KLL options | `stage1_logical_asap_optimization_b_pass1_offers_exact_and_kll` | passes | +| Each summary option appears with no window, a 5-min sliding window with a 1-min slide, and 1-min tumbling windows | `stage1_logical_asap_optimization_b_window_composition_adds_sliding_and_tumbling_per_option` | ignored: tumbling #601, sliding no PR yet | +| Window parameters are legal: L divides W; s divides L and the evaluation interval; a tumbling length divides W and the interval | `stage1_logical_asap_optimization_b_window_parameters_are_legal` | passes (vacuously today) | +| Sliding with L = W has no merge; tumbling merges before the estimate | `stage1_logical_asap_optimization_b_merge_only_where_the_window_form_needs_it` | ignored: tumbling #601, sliding no PR yet | +| A window form that merges uses a mergeable summary (KLL, DDSketch), in both patterns | `stage1_logical_asap_optimization_window_merges_use_mergeable_summaries` | passes (vacuously today) | +| One selected, cheapest valid, each node charged once per consuming query | `stage3_plan_selection_b_selects_cheapest_valid` | passes | + +**Reading window summaries.** The IR has no window summary yet, so tests read +it through `planner_layering_common::window_form(dag, build)`, which returns +`WindowForm::None` today. The window-composition implementer makes it return +`Sliding { length_ms, slide_ms }`, `Tumbling { length_ms }` or +`ExponentialHistogram { horizon_ms }` from the new IR. Merges are read as +`SummaryMerge` nodes between the build and the `SummaryEstimate`. + +## Ambiguities and conflicts with the planner + +1. **Pass 1 families.** The doc offers exact and KLL; the planner also offers + DDSketch. Tests check presence of exact and KLL, and require window forms + for every summary option. +2. **Exact in window forms.** Pattern B counts 2 × 3 = 6, so the exact option + also gets sliding and tumbling forms. Tumbling needs a mergeable exact + state for quantiles (the retained values), which the planner has no + accumulator for. Tests do not require window forms for exact. +3. **Groupings.** "One candidate for every way of grouping two or more queries + onto shared window summaries" leaves open whether a grouping is a set + partition (several shared summaries at once) and how ungrouped queries + combine with their Pass 1 options. Tests require only every pair and all + five. +4. **Yearly tumbling windows** "would also work" for Pattern A. The doc does + not say whether Pass 2 generates them. Tests neither require nor forbid + them. +5. **EH accuracy.** A boundary inside an EH bucket is approximate. Whether the + accuracy model charges that against ε = 0.005 is not stated; not tested. +6. **`as_of` and `offset`.** The table gives `as_of` = T − 1 y and the query + text has `offset 1y`. We read `as_of` as the window's end (the result of + the offset at T), not as a second shift. The frontend lowers the offset + from the text. +7. **Plan selection statistics.** Time-shifted scans are priced as one minute + of data instead of years, and a 1-y range over a shared 5-y scan passes + all of its rows (#632). Once Pass 2 shares the scan (#588), these + estimates decide whether sharing looks cheaper, so the invariant test + waits on both. +8. **Identical-expression sharing.** Not mentioned by the doc. Once + implemented (#588), the planner can share the `latency_ms` scan and the + 2-y time shift read by q4 and q5 (the shift sits below the range). +9. **Additional sliding forms.** L shorter than W (merged) is allowed by the + Pass 2 rules but not listed for Pattern B. Tests allow extra forms. +10. **Latency.** Pattern B's ≤ 200 ms is not checked by any test; plan + selection does not check latency yet. diff --git a/docs/design_docs/proposals/planner-layering-example4-acceptance.md b/docs/design_docs/proposals/planner-layering-example4-acceptance.md new file mode 100644 index 00000000..8468e68f --- /dev/null +++ b/docs/design_docs/proposals/planner-layering-example4-acceptance.md @@ -0,0 +1,152 @@ +# Planner layering, Example 4: acceptance spec + +Audience: planner designers and the implementers of materialization in +physical ASAP-aware optimization and its pricing in plan selection. +Source: [planner-layering.md](planner-layering.md), "Example 4: Materialization +of window summaries in physical planning" and the Materialization section. +Tests: `crates/integration-tests/tests/planner_layering_example4.rs` (helpers +in `tests/planner_layering_common/`; workloads in +`asap_devtools::planner_layering_examples`). + +This spec was written by a test designer who does not implement the stages. +Tests run the stages through `planner_layering_common::run_stages`. A test +that needs a missing feature is `#[ignore]`d with the PR that delivers it. + +## Workloads + +Example 3's two patterns ([its spec](planner-layering-example3-acceptance.md)), +with Pattern A varied as the doc does: + +| Variant | Recurrence | Predictability | `arrival` | +|---|---|---|---| +| A, as given | batch, `invocations: 1` at T | `ad_hoc` | `mixed` | +| A, monthly | repeating every 30 days, window ending at each run | `Predictable { known_at: T }` | `mixed` | +| A, at rest | batch, `invocations: 1` at T | `ad_hoc` | `at_rest` (no ingestion rate) | +| B | repeating every 1 min | `Predictable` | `continuously_ingesting` | + +## Expected physical candidates + +Only the physical candidates of the shared logical candidate are specified; +every other logical candidate gets its own the same way (not counted). + +**Pattern A, shared EH of KLLs (read by q1–q5).** + +| Candidate | Materialization of the EH | Expected | +|---|---|---| +| A1 | query time, kept for the batch | generated; selected as given | +| A2 | ingestion time, old data backfilled once | generated unless `at_rest`; gains with monthly recurrence | +| A3 | not materialized: each query rebuilds it | generated; costs more than A1 | + +**Pattern B, 1-min tumbling KLLs.** + +| Candidate | Materialization of the tumbling KLLs | Expected | +|---|---|---| +| B1 | ingestion time, kept 5 min; merge 5 + p99 at query time | generated; usually selected | +| B2 | not materialized: rebuild all 5 from 5 min of raw samples | generated; costs at least B1 and B3 | +| B3 | query time, kept 5 min: build only the newest | generated; no ingestion-time node | + +**Pattern B, sliding window (L = 5 min, s = 1 min).** Ingestion time or query +time, both kept. Not materializing it is the no-window plan, so it is not a +separate option. + +**Today:** no window summaries, and physical ASAP-aware optimization assigns +every sub-DAG `NotMaterialized` (#667's options exist in the IR; their +enumeration is #604), so none of these candidates exists. + +## Invariants and tests + +**Reading materialization.** The physical DAG records each node's +`Materialization` (#667). Tests read it through +`planner_layering_common::materialization(p, node)` without the storage +medium (`MatAtIngestion`, `MatAtQuery`, `NotMaterialized`), and the kept +event-time span through `retention_ms(p, node)`, which returns `None` until +the IR records retention. + +### Constraints over every candidate + +| Invariant | Test | Status | +|---|---|---| +| Every node upstream of an ingestion-time node also runs at ingestion time (all variants) | `stage2_physical_asap_optimization_ingestion_time_upstream_is_ingestion_time` | passes (vacuously today) | +| With data at rest, no node runs at ingestion time | `stage2_physical_asap_optimization_at_rest_runs_nothing_at_ingestion_time` | passes (vacuously today) | +| A materialized output covers its consumers: retention ≥ max over its readers of `lookback` + offset | `stage2_physical_asap_optimization_materialized_output_covers_its_consumers` | passes (vacuously today) | + +### Pattern A + +| Invariant | Test | Status | +|---|---|---| +| The shared EH yields exactly A1, A2, A3 | `stage2_physical_asap_optimization_a_shared_eh_has_three_materialization_options` | ignored: #628, #634, #604, #625 | +| At rest, A2 is not generated; A1 and A3 remain | `stage2_physical_asap_optimization_a_at_rest_drops_the_ingestion_time_option` | ignored: #628, #634, #604, #625 | +| A materialized EH is one build node read by all five queries, priced once | `stage2_physical_asap_optimization_a_materialized_eh_is_built_once_for_all_consumers` | ignored: #628, #634, #604, #625 | +| A3 costs strictly more than A1 (five builds instead of one: a `NotMaterialized` node is charged per consuming query) | `stage3_plan_selection_a_rebuilding_per_query_costs_more_than_building_once` | ignored: #628, #634, #604, #625 | +| As given, A1 is the cheapest of the three | `stage3_plan_selection_a_once_adhoc_prefers_the_query_time_eh` | ignored: #628, #634, #604, #625 | +| Monthly, A2's cost relative to A1 drops: cost(A2)/cost(A1) monthly < as given | `stage3_plan_selection_a_monthly_amortizes_ingestion_time_maintenance` | ignored: #628, #634, #604, #625 | + +### Pattern B + +| Invariant | Test | Status | +|---|---|---| +| The tumbling KLL candidate yields exactly B1, B2, B3 | `stage2_physical_asap_optimization_b_tumbling_kll_has_three_materialization_options` | ignored: #601, #604, #625 | +| B1 builds at ingestion time; its merge and estimate run at query time | `stage2_physical_asap_optimization_b_b1_builds_at_ingestion_and_merges_at_query_time` | ignored: #601, #604 | +| B3 runs nothing at ingestion time | `stage2_physical_asap_optimization_b_b3_keeps_query_time_windows` | ignored: #601, #625 | +| The sliding KLL yields exactly ingestion time and query time, kept | `stage2_physical_asap_optimization_b_sliding_kll_has_two_materialization_options` | ignored: sliding windows (no PR yet), #604 | +| B2 costs at least B1 and B3 | `stage3_plan_selection_b_rebuilding_every_window_costs_most` | ignored: #601, #604, #625 | +| With the built-in models, B1 is the cheapest of the three | `stage3_plan_selection_b_prefers_ingestion_time_tumbling_windows` | ignored: #601, #604, #625 | + +### Selection direction, relative to the cost model + +* A3 > A1 and B2 ≥ B1, B3 hold under any model that charges each build: A3 + and B2 do strictly more of the same work. +* A1 ≤ A2 as given, and B1 ≤ B3, depend on the model. The doc's reasons are + that A2 maintains years of history for one batch, and that B3 adds the + newest build to every evaluation and needs raw data at query time. The + tests assert the doc's direction under the built-in models; if the model + prices an ingestion-time update above a query-time one, B3 may win and the + doc allows it ("B3 can win when ingestion-time work is expensive"). +* Monthly recurrence is tested as a ratio, so it does not depend on whether A2 + wins outright ("A2 can win"). + +## Ambiguities and conflicts with the planner + +1. **Charging A3 (decided).** A3 is one logical EH node that is not + materialized, so it is rebuilt by each of its five consumers. Plan + selection charges a `NotMaterialized` node once per consuming query + (`NodeCost::multiplier`) and a materialized node once; physical ASAP-aware + optimization does not duplicate the node. So A3 costs five EH builds and + A1 one, as the tests require. The A3 tests stay ignored only because the + shared EH and its materialization options do not exist yet. +2. **Recurrence input.** Materialization depends on `recurrence`, + `predictability` and `arrival`, but `optimize_physical_asap` takes only + the logical candidate and `select_plan` only the accuracy targets and + data workload. They need the per-entry recurrence for A2's amortization + and for B1 (#604). +3. **Monthly variant.** The doc does not say whether each monthly run still + ends at T or at its own run time, nor give `known_at`. Tests use a + repeating entry whose window ends at each run, known at T. +4. **Retention unit.** "Kept 5 min" is read as the event-time span the stored + state covers. For A1, which lives only for the batch, that is 5 y of + history. The test requires retention ≥ each reader's `lookback` + offset. +5. **Backfill.** A2 backfills old data once. Whether that one-time cost is + charged to the plan, and how it is amortized, is not stated; not tested. +6. **Pruning A2 at rest.** "A2 is not generated" is read as physical + ASAP-aware optimization not generating it, not a plan selection + rejection. +7. **Upstream of an ingestion-time node.** The constraint covers the scan and + range too, so an ingestion-time build's whole input chain must be marked + ingestion time in the physical DAG. +8. **Other logical candidates.** "Every other logical candidate gets its own + physical candidates the same way" gives no counts; tests count only the + shared candidate's options. +9. **B3 retention.** The doc says B3 keeps 5 min but uses "4 kept" windows + plus the newest. The test asks for 5 min (the query window). + +## Devtool + +No `--example planner-layering-4`. Pattern A and B already run as +`planner-layering-3a` and `3b`, and until physical ASAP-aware optimization +materializes something, Example 4's variants plan identically. A useful id +needs: + +* Pass 2 window composition and materialization enumeration (#601, #604); +* the workload's recurrence passed to physical ASAP-aware optimization; +* the variant (as given, monthly, at rest) selectable in `stage_pipeline`; +* the viewer showing each node's materialization and retention. diff --git a/docs/user_guide_docs/stage-viewer.md b/docs/user_guide_docs/stage-viewer.md index ff324c19..b13891cd 100644 --- a/docs/user_guide_docs/stage-viewer.md +++ b/docs/user_guide_docs/stage-viewer.md @@ -68,10 +68,10 @@ plan wins. | Tab | Workload | What to look for | | --- | --- | --- | -| 1 · dashboard panels | Two PromQL panels over 1M counter series every 10 s: an exact per-job rate, and the approximate top-10 series per job | The exact query stays exact, the top-k uses one CMS+heap over the raw samples, and both read one shared scan | -| 2 · SQL flow statistics | Distinct sources, entropy and L2 of per-source counts over the last minute of flows | Every UnivMon plan is invalid with the reason "no accuracy model for UnivMon", so an exact plan wins | -| 3a · historical p99 batch | Five p99 reports over 1–5 years, run once | The cheapest plan shares one input scan across the five KLL sketches | -| 3b · live p99 panel | p99 over the last 5 min every minute, 1M series | Keeping KLL panes from ingestion time costs more memory than building the sketch at each evaluation, and rebuilding all panes at query time is over the 200 ms latency bound | +| 1 · dashboard panels | Two PromQL panels over 1M counter series every 10 s: an exact per-job rate, and the approximate top-10 series per job | The exact query stays exact; the CMS+heap and CountSketch+heap top-k plans are rejected with their reasons, so the all-exact plan wins; each query reads its own scan until cross-query sharing lands | +| 2 · SQL flow statistics | Distinct sources, entropy and L2 of per-source counts over the last minute of flows | All 80 plans compose and are priced; none is pruned. The SQL frontend does not yet recognize entropy and L2, so Q2 and Q3 are per-source counts with exact, Count-Min, CountSketch or UnivMon options. 68 plans are invalid: HLL for Q1 because its bound has "failure probability None", and UnivMon for Q1 or Count-Min or CountSketch for Q2 and Q3 because "the output has no guarantee, and the deployment has no evidence that it meets target". 12 are valid: Q1 exact, Theta or KMV with Q2 and Q3 each exact or UnivMon. P1 (all exact) is selected as the first of four plans tied at 52.80111 ms (with UnivMon on Q2, Q3 or both); Theta or KMV on Q1 costs 2·10⁻⁶ ms more | +| 3a · historical p99 batch | Five p99 reports over 1–5 years, run once | All 243 plans are priced. Nothing is shared yet: each query scans its own interval and builds its own sketch at query time. P122, KLL for all five queries, is selected: it is the first of 32 plans tied at the lowest cost, because KLL and DDSketch cost the same for every query; exact on any query costs more. The all-exact plan costs about 16% more | +| 3b · live p99 panel | p99 over the last 5 min every minute, 1M series | P2, KLL built from the window's raw samples at each evaluation, is selected; DDSketch costs the same and exact about 14% more. Everything runs at query time and nothing is materialized. KLL panes kept from ingestion time and the 200 ms latency check come with the window-summary PRs | | 4a · monthly p99 reports | Example 3a repeated monthly | The same plan, with its cost amortized over a month | | 4b · panes that pay off | p99 over the last hour every 10 min, 1k series sampled every second, raw data not kept by the deployment | Maintaining six 10-min KLL panes at ingestion time wins | @@ -208,7 +208,7 @@ cargo run -p asap-devtools --bin stage_pipeline -- \ | `--epsilon`, `--delta` | Accuracy for every query; without them the queries are exact | | `--interval-ms ` | Sample interval (default 15000) | | `--example ` | A built-in example instead of `--promql`: `planner-layering-1`, `-2`, `-3a`, `-3b`, `-4a` or `-4b` | -| `--max-candidates ` | How many plans the document carries, cheapest first (default 64). Stage 3 still prices every plan. | +| `--max-displayed ` | How many plans the document shows (default 64), in enumeration order, plus the selected plan. The planner still optimizes and prices every plan, so the selection does not depend on it. | | `--out ` | Where to write the document | Open the document with **Open stage document…**, by dropping the file on the