From d894ded4c0ea5323f2759d9f4dcc8f34c112ada6 Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Thu, 19 Nov 2026 12:06:37 +0100 Subject: [PATCH 1/2] examples: add progress_tick benchmark --- timely/examples/progress_tick.rs | 71 ++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 timely/examples/progress_tick.rs diff --git a/timely/examples/progress_tick.rs b/timely/examples/progress_tick.rs new file mode 100644 index 000000000..044cd54ad --- /dev/null +++ b/timely/examples/progress_tick.rs @@ -0,0 +1,71 @@ +//! Measures the cost of advancing input frontiers through dataflows that carry no data. +//! +//! Usage: progress_tick [dataflows] [regions] [operators] [ticks] [rounds] +//! +//! Builds `dataflows` dataflows, each a chain of `regions` regions that each hold a chain of +//! `operators` frontier-observing operators. Each tick advances every input by one and steps +//! the worker until every probe has caught up. + +use std::time::Instant; + +use timely::dataflow::operators::generic::Operator; +use timely::dataflow::channels::pact::Pipeline; +use timely::dataflow::operators::{Input, Probe}; +use timely::dataflow::{InputHandle, ProbeHandle}; + +fn main() { + let mut args = std::env::args().skip(1).map(|x| x.parse::().unwrap()); + let dataflows = args.next().unwrap_or(100); + let regions = args.next().unwrap_or(5); + let operators = args.next().unwrap_or(10); + let ticks = args.next().unwrap_or(200); + let rounds = args.next().unwrap_or(5); + + timely::execute_directly(move |worker| { + let mut inputs = Vec::new(); + let mut probe = ProbeHandle::new(); + for _ in 0..dataflows { + let mut input: InputHandle = InputHandle::new(); + worker.dataflow(|scope| { + let mut stream: timely::dataflow::StreamVec<_, u64> = scope.input_from(&mut input); + for _ in 0..regions { + stream = scope.clone().region(|inner| { + use timely::dataflow::operators::{Enter, Leave}; + let mut s = stream.enter(inner); + for _ in 0..operators { + s = s.unary_frontier(Pipeline, "Frontier", |_, _| { + move |(input, _frontier), output| { + input.for_each(|time, data| { + output.session(&time).give_container(data); + }); + } + }); + } + s.leave(scope) + }); + } + stream.probe_with(&mut probe); + }); + inputs.push(input); + } + + let mut time = 0u64; + let mut tick = |worker: &mut timely::worker::Worker| { + time += 1; + for input in inputs.iter_mut() { input.advance_to(time); } + while probe.less_than(&time) { worker.step(); } + }; + for _ in 0..20 { tick(worker); } + + let ops = dataflows * regions * operators; + let mut best = f64::INFINITY; + for _ in 0..rounds { + let start = Instant::now(); + for _ in 0..ticks { tick(worker); } + let ns = start.elapsed().as_nanos() as f64 / (ticks * ops) as f64; + best = best.min(ns); + println!("{ns:.1} ns/operator/tick"); + } + println!("best {best:.1} ns/operator/tick ({ops} operators)"); + }); +} From 48a45fa8585018610b950d7bd82dd0e8d90cb195 Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Thu, 19 Nov 2026 12:17:07 +0100 Subject: [PATCH 2/2] Group propagated frontier changes by operator `Tracker::propagate_all` collected every frontier change of a scope in one `ChangeBatch<(Location, T)>`. The worklist produces changes in time order and the batch sorts by location, so the batch sorted its whole contents each time it doubled and once more when drained. For a scope of 200 operators that sorting was 43% of the cost of advancing an input frontier. The tracker now keeps one small batch per operator, and a list of the operators that have changes. Changes still consolidate per location and time, so transient changes in cyclic scopes cancel as before, but no sort spans more than one operator. The operators listed are distinct, which also removes the sort of `maybe_shutdown`. `Tracker::pushed` now returns `PushedChanges`, which hands out the changes one operator at a time. This is a breaking change for callers of `pushed`. Measured with the new `progress_tick` example, which advances the inputs of dataflows that carry no data and reports nanoseconds per operator per tick. Median of five interleaved runs pinned to one core, x86-64: dataflows x regions x operators before after 1 x 1 x 200 332.7 252.0 -24.3% 100 x 5 x 10 829.2 722.6 -12.9% 100 x 1 x 200 729.7 623.3 -14.6% 1 x 1 x 100000 1000.5 719.9 -28.0% Co-Authored-By: Claude Fable 5.2 --- timely/src/progress/reachability.rs | 80 ++++++++++++++++++++++------- timely/src/progress/subgraph.rs | 42 +++++++-------- 2 files changed, 83 insertions(+), 39 deletions(-) diff --git a/timely/src/progress/reachability.rs b/timely/src/progress/reachability.rs index 26af56f99..1f415f4a6 100644 --- a/timely/src/progress/reachability.rs +++ b/timely/src/progress/reachability.rs @@ -35,13 +35,14 @@ //! // Propagate changes; until this call updates are simply buffered. //! tracker.propagate_all(); //! -//! let mut results = -//! tracker -//! .pushed() -//! .0 -//! .drain() -//! .filter(|((location, time), delta)| location.is_target()) -//! .collect::>(); +//! let mut results = Vec::new(); +//! let (pushed, _) = tracker.pushed(); +//! while let Some((node, changes)) = pushed.pop() { +//! for ((port, time), delta) in changes.drain() { +//! let location = Location { node, port }; +//! if location.is_target() { results.push(((location, time), delta)); } +//! } +//! } //! //! results.sort(); //! @@ -58,13 +59,14 @@ //! // Propagate changes; until this call updates are simply buffered. //! tracker.propagate_all(); //! -//! let mut results = -//! tracker -//! .pushed() -//! .0 -//! .drain() -//! .filter(|((location, time), delta)| location.is_target()) -//! .collect::>(); +//! let mut results = Vec::new(); +//! let (pushed, _) = tracker.pushed(); +//! while let Some((node, changes)) = pushed.pop() { +//! for ((port, time), delta) in changes.drain() { +//! let location = Location { node, port }; +//! if location.is_target() { results.push(((location, time), delta)); } +//! } +//! } //! //! results.sort(); //! @@ -417,7 +419,7 @@ pub struct Tracker { worklist: BinaryHeap>, /// Buffer of consequent changes. - pushed_changes: ChangeBatch<(Location, T)>, + pushed_changes: PushedChanges, /// Compiled summaries from each internal location (not scope inputs) to each scope output. output_changes: Vec>, @@ -434,6 +436,44 @@ pub struct Tracker { logger: Option>, } +/// Changes to the frontiers at operator ports, grouped by operator. +pub struct PushedChanges { + /// Changes for each operator, by port and time. + changes: Vec>, + /// Operators that have received changes since they were last popped, each listed once. + pending: Vec, + /// For each operator, whether it is present in `pending`. + is_pending: Vec, +} + +impl PushedChanges { + fn new(nodes: usize) -> Self { + Self { + changes: (0..nodes).map(|_| ChangeBatch::new()).collect(), + pending: Vec::new(), + is_pending: vec![false; nodes], + } + } + + fn update(&mut self, location: Location, time: T, diff: i64) { + if !self.is_pending[location.node] { + self.is_pending[location.node] = true; + self.pending.push(location.node); + } + self.changes[location.node].update((location.port, time), diff); + } + + /// Removes an operator with changes, and returns it with its changes. + /// + /// The changes are consolidated, and may be empty if they cancelled. Changes that are + /// not drained remain, and are presented again with the next changes for the operator. + pub fn pop(&mut self) -> Option<(usize, &mut ChangeBatch<(Port, T)>)> { + let node = self.pending.pop()?; + self.is_pending[node] = false; + Some((node, &mut self.changes[node])) + } +} + /// Target and source information for each operator. pub struct PerOperator { /// Port information for each target. @@ -610,6 +650,8 @@ impl Tracker { let scope_outputs = builder.shape[0].0; let output_changes = vec![ChangeBatch::new(); scope_outputs]; + let pushed_changes = PushedChanges::new(per_operator.len()); + let tracker = Tracker { nodes, @@ -620,7 +662,7 @@ impl Tracker { target_changes: ChangeBatch::new(), source_changes: ChangeBatch::new(), worklist: BinaryHeap::new(), - pushed_changes: ChangeBatch::new(), + pushed_changes, output_changes, total_counts: 0, logger, @@ -734,7 +776,7 @@ impl Tracker { self.worklist.push(Reverse((new_time, source, diff))); } } - self.pushed_changes.update((location, time), diff); + self.pushed_changes.update(location, time, diff); } } // Update to an operator output. @@ -755,7 +797,7 @@ impl Tracker { diff, ))); } - self.pushed_changes.update((location, time), diff); + self.pushed_changes.update(location, time, diff); } }, }; @@ -769,7 +811,7 @@ impl Tracker { } /// A mutable reference to the pushed results of changes. - pub fn pushed(&mut self) -> (&mut ChangeBatch<(Location, T)>, &[PerOperator]) { + pub fn pushed(&mut self) -> (&mut PushedChanges, &[PerOperator]) { (&mut self.pushed_changes, &self.per_operator) } diff --git a/timely/src/progress/subgraph.rs b/timely/src/progress/subgraph.rs index ee014452d..33a69522b 100644 --- a/timely/src/progress/subgraph.rs +++ b/timely/src/progress/subgraph.rs @@ -493,30 +493,32 @@ where // Drain propagated information into shared progress structure. let (pushed, operators) = self.pointstamp_tracker.pushed(); - for ((location, time), diff) in pushed.drain() { - self.maybe_shutdown.push(location.node); - // Targets are actionable, sources are not. - if let crate::progress::Port::Target(port) = location.port { - // Activate based on expressed frontier interest for this input. - let activate = match self.children[location.node].notify[port] { - FrontierInterest::Always => true, - FrontierInterest::IfCapability => { operators[location.node].cap_counts > 0 } - FrontierInterest::Never => false, - }; - if activate { self.temp_active.push(Reverse(location.node)); } - - // Keep this current independent of the interest. - self.children[location.node] - .shared_progress - .borrow_mut() - .frontiers[port] - .update(time, diff); + while let Some((node, changes)) = pushed.pop() { + let child = &self.children[node]; + // Changes that cancelled leave nothing to report, and no reason to revisit the operator. + if !changes.is_empty() { self.maybe_shutdown.push(node); } + for ((port, time), diff) in changes.drain() { + // Targets are actionable, sources are not. + if let crate::progress::Port::Target(port) = port { + // Activate based on expressed frontier interest for this input. + let activate = match child.notify[port] { + FrontierInterest::Always => true, + FrontierInterest::IfCapability => { operators[node].cap_counts > 0 } + FrontierInterest::Never => false, + }; + if activate { self.temp_active.push(Reverse(node)); } + + // Keep this current independent of the interest. + child + .shared_progress + .borrow_mut() + .frontiers[port] + .update(time, diff); + } } } // Consider scheduling each recipient of progress information to shut down. - self.maybe_shutdown.sort_unstable(); - self.maybe_shutdown.dedup(); for child_index in self.maybe_shutdown.drain(..) { let child_state = self.pointstamp_tracker.node_state(child_index); let frontiers_empty = child_state.targets.iter().all(|x| x.implications.is_empty());