Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
45 commits
Select commit Hold shift + click to select a range
7143a97
feat: lower SQL through unified operators and scalars
zzylol Oct 2, 2026
888c8a2
fix: integrate with #614 (DataFusion 54) in the unified SQL lowering
zzylol Oct 5, 2026
32e4dad
style: apply cargo fmt
zzylol Oct 6, 2026
1b69820
fix(sql): fold numeric casts of literals as DataFusion does
zzylol Oct 10, 2026
148c49e
feat(promql): lower queries to unified operator and scalar IR
zzylol Oct 2, 2026
1a1e45f
feat(ir): restore execution timing assignment for physical plans
zzylol Oct 3, 2026
0315635
feat(ir): export timed plans as PhysicalASAPDAG
zzylol Oct 3, 2026
f0603a2
feat(runtime): compile unified operator and scalar graphs
zzylol Oct 2, 2026
03e4449
feat(ir): give physical ASAP DAGs one root per batch query
zzylol Oct 3, 2026
64dc8a1
fix: integrate with #614 (DataFusion 54): let chrono resolve to DF 54…
zzylol Oct 5, 2026
931b991
refactor(runtime): read Operator<NodeId> payloads from the physical A…
zzylol Oct 6, 2026
f306a96
refactor(ir): follow derived summary coverage in timing and physical …
zzylol Oct 7, 2026
3a2f064
refactor(planner): plan one shared unified workload DAG
zzylol Oct 3, 2026
69f6c59
feat(planner): declare whole-source coverage on planned summary states
zzylol Oct 3, 2026
bc1c32e
feat(planner): export a planned workload as one physical ASAP DAG
zzylol Oct 3, 2026
7842beb
refactor(planner): read Operator<NodeId> payloads after the cutover
zzylol Oct 6, 2026
f4569fe
fix(planner): build planned summary coverage from the subtree
zzylol Oct 6, 2026
c78c372
refactor(planner): follow derived summary coverage
zzylol Oct 7, 2026
27fd221
test(integration): derive summary coverage from SQL and PromQL queries
zzylol Oct 10, 2026
e64f3ea
refactor(ir): remove the legacy QueryExpr and SummaryExpr IRs
zzylol Oct 3, 2026
17a0110
docs: describe the unified operator IR instead of the removed legacy IRs
zzylol Oct 3, 2026
8cbd29c
chore(tools): align the DAG viewer with the unified operator kinds
zzylol Oct 3, 2026
dc59f19
refactor(planner): remove the summary maintenance lifecycle
zzylol Oct 3, 2026
417a678
refactor(ir): default to query-time materialization
zzylol Oct 3, 2026
1cfa8e4
docs: drop the summary maintenance lifecycle and document the query-t…
zzylol Oct 3, 2026
e06e110
chore(tools): name OperatorNode in viewer comments
zzylol Oct 3, 2026
f21c3a1
refactor: finish the Operator<NodeId> port after the legacy cleanup
zzylol Oct 6, 2026
d96214a
feat(planner): enumerate phase-free local logical alternatives
zzylol Oct 3, 2026
41d8fff
docs(planner): say local candidates have no execution timing assigned
zzylol Oct 3, 2026
863ff25
test(planner): call the promoted PromQL workload lowering
zzylol Oct 3, 2026
7c8ae7d
test(planner): validate local candidates with validate_structure afte…
zzylol Oct 8, 2026
a6d5399
feat(planner): compose a whole-workload candidate from Pass 1 choices
zzylol Oct 3, 2026
711bbd0
feat(devtools): stage_pipeline writes real Stage 0/1 viewer documents
zzylol Oct 3, 2026
1b2f615
fix(planner): adapt Stage 1 candidates to the promoted frontend and e…
zzylol Oct 3, 2026
9c4af37
fix(ir): name a finalized exact accumulator after the aggregate it re…
zzylol Oct 3, 2026
0809a95
fix(planner): rank Top-K summary items by the PromQL series identity …
zzylol Oct 3, 2026
1b27d85
test(devtools): regenerate the Stage 0/1 fixture after the accumulato…
zzylol Oct 3, 2026
db2f3c4
refactor(planner): derive local candidate coverage; write logical sta…
zzylol Oct 8, 2026
348b8ab
feat(planner): #509 Stage 2 physical candidates and Stage 3 plan sele…
zzylol Oct 3, 2026
a1cfba6
feat(devtools): stage_pipeline emits all four stages for Example 1
zzylol Oct 3, 2026
bd70f05
refactor(planner): read Operator<NodeId> payloads in Stage 2/3 after …
zzylol Oct 8, 2026
6d6b8dc
test: acceptance spec and pending tests for planner-layering Example 1
zzylol Oct 3, 2026
061aa22
test: run the Example 1 acceptance tests against the real stages
zzylol Oct 3, 2026
4f6e4ba
test: read Example 1 logical stages as flat DAGs after the export rem…
zzylol Oct 8, 2026
bd25cf4
refactor(types): arrange asap-types modules by #511 section (#572 B1)
zzylol Oct 3, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
14 changes: 14 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
members = [
"crates/asap-physical-operators",
"crates/types",
"crates/frontend-common",
"crates/sql-function-catalog",
"crates/asap-aware-mapping",
"crates/frontend-promql",
Expand Down
10 changes: 4 additions & 6 deletions crates/asap-aware-mapping/src/accuracy/composition.rs
Original file line number Diff line number Diff line change
Expand Up @@ -448,9 +448,7 @@ fn composed_provenance(
}

pub(super) fn exact_operation_rule(operation: &ExactOperation) -> Option<CompositionOperator> {
let ExactOperation::Aggregate { measures, .. } = operation else {
return None;
};
let ExactOperation::Aggregate { measures, .. } = operation;
match measures.as_slice() {
[intent] => crate::function_rules::function_rules(intent).map(|rules| rules.accuracy),
// The remaining functions are exact over exact samples, but have
Expand Down Expand Up @@ -707,7 +705,7 @@ pub(super) fn propagate(
#[cfg(test)]
mod tests {
use super::*;
use asap_types::pre_asap::AggIntent;
use asap_types::ir::operator::AggIntent;
fn abs(bound: f64, delta: f64) -> ResultGuarantee {
ResultGuarantee {
metric: ErrorMetric::AbsoluteValue,
Expand Down Expand Up @@ -1032,11 +1030,11 @@ mod tests {
#[test]
fn counter_functions_have_distinct_definition_rules() {
let operation = |intent| ExactOperation::Aggregate {
reduction: asap_types::pre_asap::Reduction::PerEntity,
reduction: asap_types::ir::operator::Reduction::PerEntity,
measures: vec![intent],
output_names: vec![],
having: None,
filters: vec![],
having: None,
};
assert_eq!(
DefaultAccuracyModel.exact_operation_rule(&operation(AggIntent::Rate)),
Expand Down
8 changes: 4 additions & 4 deletions crates/asap-aware-mapping/src/accuracy/estimators/cms.rs
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,8 @@ mod tests {
#[test]
fn local_guarantee_inverts_frequency_sizing() {
use crate::replacement::default_size_params;
use asap_types::post_asap::{GroupingStrategy, SketchKind};
let c = asap_types::pre_asap::agg_intent::default_cardinality();
use asap_types::ir::schema::{GroupingStrategy, SketchKind};
let c = asap_types::ir::operator::agg_intent::default_cardinality();
let params = default_size_params(SketchAlgorithm::Cms, &c, 0.01, 0.001);
let g = DefaultAccuracyModel
.local_guarantee(
Expand All @@ -65,8 +65,8 @@ mod tests {
}

#[test]
fn heap_readout_retains_frequency_metric() {
use asap_types::post_asap::{GroupingStrategy, SketchKind};
fn heap_evaluation_retains_frequency_metric() {
use asap_types::ir::schema::{GroupingStrategy, SketchKind};
let cms_heap = SketchParams::CmsWithHeap {
width: 272,
depth: 5,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,8 @@ mod tests {
#[test]
fn count_sketch_uses_an_l2_guarantee() {
use crate::replacement::default_size_params;
use asap_types::post_asap::{GroupingStrategy, SketchKind};
use asap_types::pre_asap::agg_intent::default_cardinality;
use asap_types::ir::operator::agg_intent::default_cardinality;
use asap_types::ir::schema::{GroupingStrategy, SketchKind};
let intent = default_cardinality();
let count_sketch = default_size_params(SketchAlgorithm::CountSketch, &intent, 0.01, 0.01);
let guarantee = DefaultAccuracyModel
Expand All @@ -67,7 +67,7 @@ mod tests {
GroupingStrategy::default(),
),
&SketchStatistic::PointCount {
key: asap_types::pre_asap::expr_ir::ColumnRef::SampleValue,
key: asap_types::ir::scalar::ColumnRef::SampleValue,
value: None,
},
)
Expand Down
12 changes: 6 additions & 6 deletions crates/asap-aware-mapping/src/accuracy/estimators/hll.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
//! Estimator-specific confidence for classic HLL's linear-counting branch.
//!
//! This is conditional on independent uniform bucket hashes and an enforced
//! upper bound on distinct items in the complete readout population (including
//! upper bound on distinct items in the complete evaluation population (including
//! all merged panes). It is not an RSE-to-normal conversion or an ERP fit.

use super::*;
Expand Down Expand Up @@ -56,13 +56,13 @@ impl ClassicHllConfidence {
value: self.relative_error,
},
failure_probability: ProbabilityExpr::Constant { value: delta },
provenance: vec![GuaranteeSource::SketchReadout {
provenance: vec![GuaranteeSource::SketchEvaluation {
algorithm: "Hll".into(),
contract: "classic_hll_linear_counting_collision_bound_v1".into(),
params: serde_json::json!({"precision": precision,
"max_distinct": self.max_distinct, "relative_error": self.relative_error,
"hash_assumption": "independent_uniform_buckets",
"population_scope": "complete_readout_including_merged_panes"}),
"population_scope": "complete_evaluation_including_merged_panes"}),
query: "Cardinality".into(),
}],
})
Expand Down Expand Up @@ -187,7 +187,7 @@ mod tests {
}
}
}
/// The model's readout formula matches the actual classic estimator after merge.
/// The model's evaluation formula matches the actual classic estimator after merge.
#[test]
fn native_classic_estimator_and_merged_registers_use_the_same_contract() {
use asap_sketchlib::sketches::hll::{Classic, HyperLogLogP16};
Expand Down Expand Up @@ -231,8 +231,8 @@ mod tests {
#[test]
fn generic_rse_sizing_does_not_certify_confidence() {
use crate::replacement::default_size_params;
use asap_types::post_asap::{GroupingStrategy, SketchKind};
use asap_types::pre_asap::agg_intent::default_cardinality;
use asap_types::ir::operator::agg_intent::default_cardinality;
use asap_types::ir::schema::{GroupingStrategy, SketchKind};
let c = default_cardinality();
let params = default_size_params(SketchAlgorithm::Hll, &c, 0.01, 0.01);
let g = DefaultAccuracyModel
Expand Down
6 changes: 3 additions & 3 deletions crates/asap-aware-mapping/src/accuracy/estimators/kll.rs
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,8 @@ mod tests {
#[test]
fn local_guarantee_inverts_rank_sizing() {
use crate::replacement::default_size_params;
use asap_types::post_asap::{GroupingStrategy, SketchKind};
use asap_types::pre_asap::agg_intent::default_quantile;
use asap_types::ir::operator::agg_intent::default_quantile;
use asap_types::ir::schema::{GroupingStrategy, SketchKind};
let q = default_quantile(0.99);
let params = default_size_params(SketchAlgorithm::Kll, &q, 0.01, 0.01);
let g = DefaultAccuracyModel
Expand All @@ -69,7 +69,7 @@ mod tests {
assert_eq!(g.approximate_layer_count(), 1);
assert!(g.provenance.iter().any(|source| matches!(
source,
GuaranteeSource::SketchReadout { contract, .. }
GuaranteeSource::SketchEvaluation { contract, .. }
if contract == "apache_datasketches_kll_empirical_99_a9b42755072b"
)));
}
Expand Down
10 changes: 5 additions & 5 deletions crates/asap-aware-mapping/src/accuracy/estimators/mod.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
//! Dispatch committed estimator parameters to their accuracy models.
use super::*;
use asap_types::post_asap::GroupingStrategy;
use asap_types::pre_asap::AggIntent;
use asap_types::ir::operator::AggIntent;
use asap_types::ir::schema::GroupingStrategy;

pub mod cardinality;
pub mod cms;
Expand Down Expand Up @@ -46,7 +46,7 @@ fn bounded_guarantee(
metric,
bound: BoundExpr::Constant { value: bound },
failure_probability: delta,
provenance: vec![GuaranteeSource::SketchReadout {
provenance: vec![GuaranteeSource::SketchEvaluation {
algorithm: format!("{algorithm:?}"),
contract: contract.into(),
params: serde_json::to_value(params).unwrap_or(serde_json::Value::Null),
Expand Down Expand Up @@ -169,9 +169,9 @@ impl<'a> EstimatorAccuracy<'a> {

fn hll(&self) -> Option<hll::ClassicHllConfidence> {
let EstimatorContract::ClassicHll {
max_distinct_per_readout,
max_distinct_per_evaluation,
} = self.contract?;
hll::ClassicHllConfidence::new(max_distinct_per_readout, self.epsilon)
hll::ClassicHllConfidence::new(max_distinct_per_evaluation, self.epsilon)
}

pub(crate) fn size_params(&self, algorithm: &SketchAlgorithm) -> Option<SketchParams> {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
//! UnivMon currently certifies only its exact unit-update total readout.
//! UnivMon currently certifies only its exact unit-update total evaluation.
use super::*;

pub(super) fn guarantee(query: &SketchStatistic) -> Option<ResultGuarantee> {
Expand Down
33 changes: 12 additions & 21 deletions crates/asap-aware-mapping/src/accuracy/evidence.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,12 @@
use super::*;

/// A trusted source assertion scoped by `AccuracyEvidenceProvider` to one
/// complete readout. Choosing this variant asserts the estimator and hash
/// complete evaluation. Choosing this variant asserts the estimator and hash
/// assumptions; it must not be inferred from sampled population statistics.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum EstimatorContract {
/// Classic HLL with independent uniform bucket hashing, including merged panes.
ClassicHll { max_distinct_per_readout: u32 },
ClassicHll { max_distinct_per_evaluation: u32 },
}

/// An enforced domain for every sample of a direct quantile operand, in every
Expand All @@ -18,7 +18,7 @@ pub enum EstimatorContract {
pub struct QuantileInputDomain {
pub lower: f64,
pub upper: f64,
/// Upper bound on samples per evaluation, matching the pinned readout's
/// Upper bound on samples per evaluation, matching the pinned evaluation's
/// exact Float64 rank limit. The population must also be nonempty.
pub max_samples: u64,
pub contract: String,
Expand Down Expand Up @@ -87,31 +87,22 @@ pub struct PropagationStats {
/// Supplies typed planning-time evidence required by propagation rules.
pub trait AccuracyEvidenceProvider {
/// Trusted estimator contract for this complete aggregate expression,
/// including source, filters, grouping and all panes in each readout.
/// including source, filters, grouping and all panes in each evaluation.
/// An observed cardinality is not an enforced population bound.
fn estimator_contract(
&self,
_expression: &asap_types::pre_asap::QueryExpr,
) -> Option<EstimatorContract> {
fn estimator_contract(&self, _expression: &OperatorNode) -> Option<EstimatorContract> {
None
}

/// Enforced upper bound on distinct (partition, item) identities across a
/// complete TopK readout. Used to union-bound score errors for adaptively
/// complete TopK evaluation. Used to union-bound score errors for adaptively
/// selected candidates. Observed cardinality is not sufficient evidence.
fn topk_max_distinct_items(
&self,
_expression: &asap_types::pre_asap::QueryExpr,
) -> Option<u64> {
fn topk_max_distinct_items(&self, _expression: &OperatorNode) -> Option<u64> {
None
}

/// Proof scoped to this complete quantile expression, including its source,
/// filters, grouping and window. `None` means unknown, including emptiness.
fn quantile_input_domain(
&self,
_operand: &asap_types::pre_asap::query_expr::QueryExpr,
) -> Option<QuantileInputDomain> {
fn quantile_input_domain(&self, _operand: &OperatorNode) -> Option<QuantileInputDomain> {
None
}

Expand Down Expand Up @@ -181,8 +172,8 @@ mod tests {
let fresh = provider.propagation_stats(
&CompositionOperator::ExactSum,
&FieldDataType::ExactAggregate(
asap_types::post_asap::ExactKind::Sum,
asap_types::post_asap::ExactParams::Sum,
asap_types::ir::schema::ExactKind::Sum,
asap_types::ir::schema::ExactParams::Sum,
),
None,
);
Expand All @@ -196,8 +187,8 @@ mod tests {
.propagation_stats(
&CompositionOperator::ExactSum,
&FieldDataType::ExactAggregate(
asap_types::post_asap::ExactKind::Sum,
asap_types::post_asap::ExactParams::Sum,
asap_types::ir::schema::ExactKind::Sum,
asap_types::ir::schema::ExactParams::Sum,
),
None,
);
Expand Down
15 changes: 9 additions & 6 deletions crates/asap-aware-mapping/src/accuracy/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -20,18 +20,21 @@ pub use evidence::{
QuantileInputDomain, WorkloadAccuracyEvidence,
};

use asap_types::post_asap::{
AccuracyError, BoundExpr, CompositionOperator, ErrorMetric, ExactOperation, FieldDataType,
GuaranteeSource, ProbabilityExpr, ResultGuarantee, SketchAlgorithm, SketchParams,
SketchStatistic,
use asap_types::ir::properties::{
AccuracyError, BoundExpr, CompositionOperator, ErrorMetric, GuaranteeSource, ProbabilityExpr,
ResultGuarantee,
};
use asap_types::ir::schema::{FieldDataType, SketchAlgorithm, SketchParams, SketchStatistic};
use asap_types::ir::OperatorNode;
use asap_types::types::AccuracyTarget;

use crate::exact_composition::ExactOperation;

/// The deployment-extensible accuracy algebra. `asap-aware-mapping` ships
/// [`DefaultAccuracyModel`]; a deployment with a proof for a composition the
/// default rejects (a registered cross-metric conversion, say) implements
/// this trait and passes it to
/// [`crate::replacement::SketchAlgorithmStrategy::new_with_planning_inputs`].
/// [`crate::replacement::ASAPStrategies::new_with_planning_inputs`].
pub trait AccuracyModel {
/// The definition-registered rule for applying `operation` to an
/// approximate input. `None` means the function is exact only over exact
Expand Down Expand Up @@ -78,7 +81,7 @@ pub struct DefaultAccuracyModel;
const SATISFACTION_TOLERANCE: f64 = 1e-9;

impl DefaultAccuracyModel {
/// Derive the guarantee for the committed estimator parameters and readout.
/// Derive the guarantee for the committed estimator parameters and evaluation.
pub fn sketch_guarantee(
algorithm: &SketchAlgorithm,
params: &SketchParams,
Expand Down
Loading