diff --git a/timely/examples/progress_tick.rs b/timely/examples/progress_tick.rs new file mode 100644 index 000000000..044cd54ad --- /dev/null +++ b/timely/examples/progress_tick.rs @@ -0,0 +1,71 @@ +//! Measures the cost of advancing input frontiers through dataflows that carry no data. +//! +//! Usage: progress_tick [dataflows] [regions] [operators] [ticks] [rounds] +//! +//! Builds `dataflows` dataflows, each a chain of `regions` regions that each hold a chain of +//! `operators` frontier-observing operators. Each tick advances every input by one and steps +//! the worker until every probe has caught up. + +use std::time::Instant; + +use timely::dataflow::operators::generic::Operator; +use timely::dataflow::channels::pact::Pipeline; +use timely::dataflow::operators::{Input, Probe}; +use timely::dataflow::{InputHandle, ProbeHandle}; + +fn main() { + let mut args = std::env::args().skip(1).map(|x| x.parse::().unwrap()); + let dataflows = args.next().unwrap_or(100); + let regions = args.next().unwrap_or(5); + let operators = args.next().unwrap_or(10); + let ticks = args.next().unwrap_or(200); + let rounds = args.next().unwrap_or(5); + + timely::execute_directly(move |worker| { + let mut inputs = Vec::new(); + let mut probe = ProbeHandle::new(); + for _ in 0..dataflows { + let mut input: InputHandle = InputHandle::new(); + worker.dataflow(|scope| { + let mut stream: timely::dataflow::StreamVec<_, u64> = scope.input_from(&mut input); + for _ in 0..regions { + stream = scope.clone().region(|inner| { + use timely::dataflow::operators::{Enter, Leave}; + let mut s = stream.enter(inner); + for _ in 0..operators { + s = s.unary_frontier(Pipeline, "Frontier", |_, _| { + move |(input, _frontier), output| { + input.for_each(|time, data| { + output.session(&time).give_container(data); + }); + } + }); + } + s.leave(scope) + }); + } + stream.probe_with(&mut probe); + }); + inputs.push(input); + } + + let mut time = 0u64; + let mut tick = |worker: &mut timely::worker::Worker| { + time += 1; + for input in inputs.iter_mut() { input.advance_to(time); } + while probe.less_than(&time) { worker.step(); } + }; + for _ in 0..20 { tick(worker); } + + let ops = dataflows * regions * operators; + let mut best = f64::INFINITY; + for _ in 0..rounds { + let start = Instant::now(); + for _ in 0..ticks { tick(worker); } + let ns = start.elapsed().as_nanos() as f64 / (ticks * ops) as f64; + best = best.min(ns); + println!("{ns:.1} ns/operator/tick"); + } + println!("best {best:.1} ns/operator/tick ({ops} operators)"); + }); +} diff --git a/timely/src/progress/reachability.rs b/timely/src/progress/reachability.rs index 26af56f99..1f415f4a6 100644 --- a/timely/src/progress/reachability.rs +++ b/timely/src/progress/reachability.rs @@ -35,13 +35,14 @@ //! // Propagate changes; until this call updates are simply buffered. //! tracker.propagate_all(); //! -//! let mut results = -//! tracker -//! .pushed() -//! .0 -//! .drain() -//! .filter(|((location, time), delta)| location.is_target()) -//! .collect::>(); +//! let mut results = Vec::new(); +//! let (pushed, _) = tracker.pushed(); +//! while let Some((node, changes)) = pushed.pop() { +//! for ((port, time), delta) in changes.drain() { +//! let location = Location { node, port }; +//! if location.is_target() { results.push(((location, time), delta)); } +//! } +//! } //! //! results.sort(); //! @@ -58,13 +59,14 @@ //! // Propagate changes; until this call updates are simply buffered. //! tracker.propagate_all(); //! -//! let mut results = -//! tracker -//! .pushed() -//! .0 -//! .drain() -//! .filter(|((location, time), delta)| location.is_target()) -//! .collect::>(); +//! let mut results = Vec::new(); +//! let (pushed, _) = tracker.pushed(); +//! while let Some((node, changes)) = pushed.pop() { +//! for ((port, time), delta) in changes.drain() { +//! let location = Location { node, port }; +//! if location.is_target() { results.push(((location, time), delta)); } +//! } +//! } //! //! results.sort(); //! @@ -417,7 +419,7 @@ pub struct Tracker { worklist: BinaryHeap>, /// Buffer of consequent changes. - pushed_changes: ChangeBatch<(Location, T)>, + pushed_changes: PushedChanges, /// Compiled summaries from each internal location (not scope inputs) to each scope output. output_changes: Vec>, @@ -434,6 +436,44 @@ pub struct Tracker { logger: Option>, } +/// Changes to the frontiers at operator ports, grouped by operator. +pub struct PushedChanges { + /// Changes for each operator, by port and time. + changes: Vec>, + /// Operators that have received changes since they were last popped, each listed once. + pending: Vec, + /// For each operator, whether it is present in `pending`. + is_pending: Vec, +} + +impl PushedChanges { + fn new(nodes: usize) -> Self { + Self { + changes: (0..nodes).map(|_| ChangeBatch::new()).collect(), + pending: Vec::new(), + is_pending: vec![false; nodes], + } + } + + fn update(&mut self, location: Location, time: T, diff: i64) { + if !self.is_pending[location.node] { + self.is_pending[location.node] = true; + self.pending.push(location.node); + } + self.changes[location.node].update((location.port, time), diff); + } + + /// Removes an operator with changes, and returns it with its changes. + /// + /// The changes are consolidated, and may be empty if they cancelled. Changes that are + /// not drained remain, and are presented again with the next changes for the operator. + pub fn pop(&mut self) -> Option<(usize, &mut ChangeBatch<(Port, T)>)> { + let node = self.pending.pop()?; + self.is_pending[node] = false; + Some((node, &mut self.changes[node])) + } +} + /// Target and source information for each operator. pub struct PerOperator { /// Port information for each target. @@ -610,6 +650,8 @@ impl Tracker { let scope_outputs = builder.shape[0].0; let output_changes = vec![ChangeBatch::new(); scope_outputs]; + let pushed_changes = PushedChanges::new(per_operator.len()); + let tracker = Tracker { nodes, @@ -620,7 +662,7 @@ impl Tracker { target_changes: ChangeBatch::new(), source_changes: ChangeBatch::new(), worklist: BinaryHeap::new(), - pushed_changes: ChangeBatch::new(), + pushed_changes, output_changes, total_counts: 0, logger, @@ -734,7 +776,7 @@ impl Tracker { self.worklist.push(Reverse((new_time, source, diff))); } } - self.pushed_changes.update((location, time), diff); + self.pushed_changes.update(location, time, diff); } } // Update to an operator output. @@ -755,7 +797,7 @@ impl Tracker { diff, ))); } - self.pushed_changes.update((location, time), diff); + self.pushed_changes.update(location, time, diff); } }, }; @@ -769,7 +811,7 @@ impl Tracker { } /// A mutable reference to the pushed results of changes. - pub fn pushed(&mut self) -> (&mut ChangeBatch<(Location, T)>, &[PerOperator]) { + pub fn pushed(&mut self) -> (&mut PushedChanges, &[PerOperator]) { (&mut self.pushed_changes, &self.per_operator) } diff --git a/timely/src/progress/subgraph.rs b/timely/src/progress/subgraph.rs index ee014452d..33a69522b 100644 --- a/timely/src/progress/subgraph.rs +++ b/timely/src/progress/subgraph.rs @@ -493,30 +493,32 @@ where // Drain propagated information into shared progress structure. let (pushed, operators) = self.pointstamp_tracker.pushed(); - for ((location, time), diff) in pushed.drain() { - self.maybe_shutdown.push(location.node); - // Targets are actionable, sources are not. - if let crate::progress::Port::Target(port) = location.port { - // Activate based on expressed frontier interest for this input. - let activate = match self.children[location.node].notify[port] { - FrontierInterest::Always => true, - FrontierInterest::IfCapability => { operators[location.node].cap_counts > 0 } - FrontierInterest::Never => false, - }; - if activate { self.temp_active.push(Reverse(location.node)); } - - // Keep this current independent of the interest. - self.children[location.node] - .shared_progress - .borrow_mut() - .frontiers[port] - .update(time, diff); + while let Some((node, changes)) = pushed.pop() { + let child = &self.children[node]; + // Changes that cancelled leave nothing to report, and no reason to revisit the operator. + if !changes.is_empty() { self.maybe_shutdown.push(node); } + for ((port, time), diff) in changes.drain() { + // Targets are actionable, sources are not. + if let crate::progress::Port::Target(port) = port { + // Activate based on expressed frontier interest for this input. + let activate = match child.notify[port] { + FrontierInterest::Always => true, + FrontierInterest::IfCapability => { operators[node].cap_counts > 0 } + FrontierInterest::Never => false, + }; + if activate { self.temp_active.push(Reverse(node)); } + + // Keep this current independent of the interest. + child + .shared_progress + .borrow_mut() + .frontiers[port] + .update(time, diff); + } } } // Consider scheduling each recipient of progress information to shut down. - self.maybe_shutdown.sort_unstable(); - self.maybe_shutdown.dedup(); for child_index in self.maybe_shutdown.drain(..) { let child_state = self.pointstamp_tracker.node_state(child_index); let frontiers_empty = child_state.targets.iter().all(|x| x.implications.is_empty());