Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 71 additions & 0 deletions timely/examples/progress_tick.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
//! Measures the cost of advancing input frontiers through dataflows that carry no data.
//!
//! Usage: progress_tick [dataflows] [regions] [operators] [ticks] [rounds]
//!
//! Builds `dataflows` dataflows, each a chain of `regions` regions that each hold a chain of
//! `operators` frontier-observing operators. Each tick advances every input by one and steps
//! the worker until every probe has caught up.

use std::time::Instant;

use timely::dataflow::operators::generic::Operator;
use timely::dataflow::channels::pact::Pipeline;
use timely::dataflow::operators::{Input, Probe};
use timely::dataflow::{InputHandle, ProbeHandle};

fn main() {
let mut args = std::env::args().skip(1).map(|x| x.parse::<usize>().unwrap());
let dataflows = args.next().unwrap_or(100);
let regions = args.next().unwrap_or(5);
let operators = args.next().unwrap_or(10);
let ticks = args.next().unwrap_or(200);
let rounds = args.next().unwrap_or(5);

timely::execute_directly(move |worker| {
let mut inputs = Vec::new();
let mut probe = ProbeHandle::new();
for _ in 0..dataflows {
let mut input: InputHandle<u64, _> = InputHandle::new();
worker.dataflow(|scope| {
let mut stream: timely::dataflow::StreamVec<_, u64> = scope.input_from(&mut input);
for _ in 0..regions {
stream = scope.clone().region(|inner| {
use timely::dataflow::operators::{Enter, Leave};
let mut s = stream.enter(inner);
for _ in 0..operators {
s = s.unary_frontier(Pipeline, "Frontier", |_, _| {
move |(input, _frontier), output| {
input.for_each(|time, data| {
output.session(&time).give_container(data);
});
}
});
}
s.leave(scope)
});
}
stream.probe_with(&mut probe);
});
inputs.push(input);
}

let mut time = 0u64;
let mut tick = |worker: &mut timely::worker::Worker| {
time += 1;
for input in inputs.iter_mut() { input.advance_to(time); }
while probe.less_than(&time) { worker.step(); }
};
for _ in 0..20 { tick(worker); }

let ops = dataflows * regions * operators;
let mut best = f64::INFINITY;
for _ in 0..rounds {
let start = Instant::now();
for _ in 0..ticks { tick(worker); }
let ns = start.elapsed().as_nanos() as f64 / (ticks * ops) as f64;
best = best.min(ns);
println!("{ns:.1} ns/operator/tick");
}
println!("best {best:.1} ns/operator/tick ({ops} operators)");
});
}
80 changes: 61 additions & 19 deletions timely/src/progress/reachability.rs
Original file line number Diff line number Diff line change
Expand Up @@ -35,13 +35,14 @@
//! // Propagate changes; until this call updates are simply buffered.
//! tracker.propagate_all();
//!
//! let mut results =
//! tracker
//! .pushed()
//! .0
//! .drain()
//! .filter(|((location, time), delta)| location.is_target())
//! .collect::<Vec<_>>();
//! let mut results = Vec::new();
//! let (pushed, _) = tracker.pushed();
//! while let Some((node, changes)) = pushed.pop() {
//! for ((port, time), delta) in changes.drain() {
//! let location = Location { node, port };
//! if location.is_target() { results.push(((location, time), delta)); }
//! }
//! }
//!
//! results.sort();
//!
Expand All @@ -58,13 +59,14 @@
//! // Propagate changes; until this call updates are simply buffered.
//! tracker.propagate_all();
//!
//! let mut results =
//! tracker
//! .pushed()
//! .0
//! .drain()
//! .filter(|((location, time), delta)| location.is_target())
//! .collect::<Vec<_>>();
//! let mut results = Vec::new();
//! let (pushed, _) = tracker.pushed();
//! while let Some((node, changes)) = pushed.pop() {
//! for ((port, time), delta) in changes.drain() {
//! let location = Location { node, port };
//! if location.is_target() { results.push(((location, time), delta)); }
//! }
//! }
//!
//! results.sort();
//!
Expand Down Expand Up @@ -313,7 +315,7 @@
for outputs in summary.iter() {
for (output, summaries) in outputs.iter_ports() {
let source = Location::new_source(index, output);
for summary in summaries.elements().iter() {

Check warning on line 318 in timely/src/progress/reachability.rs

View workflow job for this annotation

GitHub Actions / Cargo clippy

`summary` shadows a previous, unrelated binding
if summary == &Default::default() {
in_degree[index_of(&source)] += 1;
}
Expand Down Expand Up @@ -417,7 +419,7 @@
worklist: BinaryHeap<Reverse<(T, Location, i64)>>,

/// Buffer of consequent changes.
pushed_changes: ChangeBatch<(Location, T)>,
pushed_changes: PushedChanges<T>,

/// Compiled summaries from each internal location (not scope inputs) to each scope output.
output_changes: Vec<ChangeBatch<T>>,
Expand All @@ -434,6 +436,44 @@
logger: Option<logging::TrackerLogger<T>>,
}

/// Changes to the frontiers at operator ports, grouped by operator.
pub struct PushedChanges<T> {
/// Changes for each operator, by port and time.
changes: Vec<ChangeBatch<(Port, T)>>,
/// Operators that have received changes since they were last popped, each listed once.
pending: Vec<usize>,
/// For each operator, whether it is present in `pending`.
is_pending: Vec<bool>,
}

impl<T: Ord> PushedChanges<T> {
fn new(nodes: usize) -> Self {
Self {
changes: (0..nodes).map(|_| ChangeBatch::new()).collect(),
pending: Vec::new(),
is_pending: vec![false; nodes],
}
}

fn update(&mut self, location: Location, time: T, diff: i64) {
if !self.is_pending[location.node] {
self.is_pending[location.node] = true;
self.pending.push(location.node);
}
self.changes[location.node].update((location.port, time), diff);
}

/// Removes an operator with changes, and returns it with its changes.
///
/// The changes are consolidated, and may be empty if they cancelled. Changes that are
/// not drained remain, and are presented again with the next changes for the operator.
pub fn pop(&mut self) -> Option<(usize, &mut ChangeBatch<(Port, T)>)> {
let node = self.pending.pop()?;
self.is_pending[node] = false;
Some((node, &mut self.changes[node]))
}
}

/// Target and source information for each operator.
pub struct PerOperator<T: Timestamp> {
/// Port information for each target.
Expand Down Expand Up @@ -610,6 +650,8 @@
let scope_outputs = builder.shape[0].0;
let output_changes = vec![ChangeBatch::new(); scope_outputs];

let pushed_changes = PushedChanges::new(per_operator.len());

let tracker =
Tracker {
nodes,
Expand All @@ -620,7 +662,7 @@
target_changes: ChangeBatch::new(),
source_changes: ChangeBatch::new(),
worklist: BinaryHeap::new(),
pushed_changes: ChangeBatch::new(),
pushed_changes,
output_changes,
total_counts: 0,
logger,
Expand Down Expand Up @@ -727,14 +769,14 @@
.implications
.update_iter(Some((time, diff)));

for (time, diff) in changes {

Check warning on line 772 in timely/src/progress/reachability.rs

View workflow job for this annotation

GitHub Actions / Cargo clippy

`diff` shadows a previous, unrelated binding

Check warning on line 772 in timely/src/progress/reachability.rs

View workflow job for this annotation

GitHub Actions / Cargo clippy

`time` shadows a previous, unrelated binding
for &(output_port, ref summary) in (&self.nodes).get(location.node).get(port_index).into_index_iter() {
if let Some(new_time) = summary.results_in(&time) {
let source = Location { node: location.node, port: Port::Source(output_port) };
self.worklist.push(Reverse((new_time, source, diff)));
}
}
self.pushed_changes.update((location, time), diff);
self.pushed_changes.update(location, time, diff);
}
}
// Update to an operator output.
Expand All @@ -747,7 +789,7 @@
.implications
.update_iter(Some((time, diff)));

for (time, diff) in changes {

Check warning on line 792 in timely/src/progress/reachability.rs

View workflow job for this annotation

GitHub Actions / Cargo clippy

`diff` shadows a previous, unrelated binding

Check warning on line 792 in timely/src/progress/reachability.rs

View workflow job for this annotation

GitHub Actions / Cargo clippy

`time` shadows a previous, unrelated binding
for new_target in (&self.edges).get(location.node).get(port_index).into_index_iter() {
self.worklist.push(Reverse((
time.clone(),
Expand All @@ -755,7 +797,7 @@
diff,
)));
}
self.pushed_changes.update((location, time), diff);
self.pushed_changes.update(location, time, diff);
}
},
};
Expand All @@ -769,7 +811,7 @@
}

/// A mutable reference to the pushed results of changes.
pub fn pushed(&mut self) -> (&mut ChangeBatch<(Location, T)>, &[PerOperator<T>]) {
pub fn pushed(&mut self) -> (&mut PushedChanges<T>, &[PerOperator<T>]) {
(&mut self.pushed_changes, &self.per_operator)
}

Expand Down Expand Up @@ -886,7 +928,7 @@
}
}
}
reverse_internal.sort_unstable_by(|x, y| (x.0, x.1).cmp(&(y.0, y.1)));

Check warning on line 931 in timely/src/progress/reachability.rs

View workflow job for this annotation

GitHub Actions / Cargo clippy

consider using `sort_unstable_by_key`

// Accumulated summaries to scope outputs, keyed by `(location, output)`.
let mut accumulated: BinaryRuns<(Location, usize), Antichain<T::Summary>> = BinaryRuns::default();
Expand Down Expand Up @@ -941,7 +983,7 @@
// Merge the batch of proposals into the accumulated summaries. Proposals are
// first collapsed per key into an antichain, so that only elements novel to
// the accumulated antichain (in an order-independent sense) seed the next round.
proposals.sort_unstable_by(|x, y| x.0.cmp(&y.0));

Check warning on line 986 in timely/src/progress/reachability.rs

View workflow job for this annotation

GitHub Actions / Cargo clippy

consider using `sort_unstable_by_key`
let mut fresh: Vec<((Location, usize), Antichain<T::Summary>)> = Vec::new();
let mut batch: Antichain<T::Summary> = Antichain::new();
let mut iter = proposals.drain(..).peekable();
Expand Down
42 changes: 22 additions & 20 deletions timely/src/progress/subgraph.rs
Original file line number Diff line number Diff line change
Expand Up @@ -493,30 +493,32 @@ where

// Drain propagated information into shared progress structure.
let (pushed, operators) = self.pointstamp_tracker.pushed();
for ((location, time), diff) in pushed.drain() {
self.maybe_shutdown.push(location.node);
// Targets are actionable, sources are not.
if let crate::progress::Port::Target(port) = location.port {
// Activate based on expressed frontier interest for this input.
let activate = match self.children[location.node].notify[port] {
FrontierInterest::Always => true,
FrontierInterest::IfCapability => { operators[location.node].cap_counts > 0 }
FrontierInterest::Never => false,
};
if activate { self.temp_active.push(Reverse(location.node)); }

// Keep this current independent of the interest.
self.children[location.node]
.shared_progress
.borrow_mut()
.frontiers[port]
.update(time, diff);
while let Some((node, changes)) = pushed.pop() {
let child = &self.children[node];
// Changes that cancelled leave nothing to report, and no reason to revisit the operator.
if !changes.is_empty() { self.maybe_shutdown.push(node); }
for ((port, time), diff) in changes.drain() {
// Targets are actionable, sources are not.
if let crate::progress::Port::Target(port) = port {
// Activate based on expressed frontier interest for this input.
let activate = match child.notify[port] {
FrontierInterest::Always => true,
FrontierInterest::IfCapability => { operators[node].cap_counts > 0 }
FrontierInterest::Never => false,
};
if activate { self.temp_active.push(Reverse(node)); }

// Keep this current independent of the interest.
child
.shared_progress
.borrow_mut()
.frontiers[port]
.update(time, diff);
}
}
}

// Consider scheduling each recipient of progress information to shut down.
self.maybe_shutdown.sort_unstable();
self.maybe_shutdown.dedup();
for child_index in self.maybe_shutdown.drain(..) {
let child_state = self.pointstamp_tracker.node_state(child_index);
let frontiers_empty = child_state.targets.iter().all(|x| x.implications.is_empty());
Expand Down
Loading