Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 14 additions & 1 deletion timely/src/progress/operate.rs
Original file line number Diff line number Diff line change
Expand Up @@ -64,11 +64,24 @@ pub trait Operate<T: Timestamp> {
}

/// The ways in which an operator can express interest in activation when an input frontier changes.
///
/// Variants are ordered from least to most interested.
#[derive(serde::Serialize, serde::Deserialize, columnar::Columnar, Ord, PartialOrd, Eq, PartialEq, Copy, Clone, Debug)]
pub enum FrontierInterest {
/// Never interested in frontier changes, as for example the `map()` and `filter()` operators.
Never,
/// Interested when the operator holds capabilities.
/// Interested when a frontier change removes an element less or equal to a held capability.
/// Held capabilities are those of this worker's instance of the operator, not those of its
/// instances on other workers.
///
/// Such a change is the only kind that can complete the time of a held capability, or a time
/// greater than one. The operator is not woken for changes entirely above its capabilities,
/// which includes all changes once the frontier has passed every capability it holds.
/// Any time the operator waits on must be greater or equal to a held capability that the
/// frontier has not yet passed; a capability the frontier has passed does not keep the
/// operator awake for later times.
UpToCapability,
/// Interested in any frontier change while the operator holds capabilities.
IfCapability,
/// Always interested in frontier changes, as for example the `probe()` and `capture()` operators.
Always,
Expand Down
29 changes: 26 additions & 3 deletions timely/src/progress/subgraph.rs
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,9 @@ where
activations.borrow_mut().activate(&self.path[..]);

// The subgraph's per-input interest is conservatively the max across all children's inputs.
// A child's `UpToCapability` interest is covered by the subgraph's own: path summaries do not
// decrease times, so a frontier change at or below a child's capability starts from a removed
// scope-input element at or below that capability's image at the scope outputs.
let max_interest = children.iter()
.flat_map(|c| c.notify.iter().copied())
.max()
Expand Down Expand Up @@ -498,9 +501,20 @@ where
// Targets are actionable, sources are not.
if let crate::progress::Port::Target(port) = location.port {
// Activate based on expressed frontier interest for this input.
let activate = match self.children[location.node].notify[port] {
let child = &mut self.children[location.node];
let activate = match child.notify[port] {
FrontierInterest::Always => true,
FrontierInterest::IfCapability => { operators[location.node].cap_counts > 0 }
// Judged against the capabilities this worker's instance holds. The tracker's
// pointstamps sum all workers and trail this worker's own downgrades, so a
// capability held here can sit above every element they show; the other
// workers' downgrades arrive later as capability changes, which change no
// frontier and so would not revisit a missed wake.
FrontierInterest::UpToCapability => {
diff < 0 && child.local_caps.as_mut().is_some_and(|caps| {
caps.iter().any(|(cap, count)| *count > 0 && time.less_equal(cap))
})
}
FrontierInterest::Never => false,
};
if activate { self.temp_active.push(Reverse(location.node)); }
Expand Down Expand Up @@ -637,6 +651,10 @@ struct PerOperatorState<T: Timestamp> {

shared_progress: Rc<RefCell<SharedProgress<T>>>,

/// Capabilities this worker's instance holds, summed over its outputs and accumulated from its
/// internals. Kept only for operators with an `UpToCapability` input.
local_caps: Option<ChangeBatch<T>>,

internal_summary: Connectivity<T::Summary>, // from initialize; moved into the reachability builder.

logging: Option<Logger>,
Expand All @@ -651,7 +669,7 @@ impl<T: Timestamp> PerOperatorState<T> {
index: 0,
id: usize::MAX,
local: false,
notify: vec![FrontierInterest::IfCapability; inputs],
notify: vec![FrontierInterest::UpToCapability; inputs],
inputs,
outputs,

Expand All @@ -660,6 +678,7 @@ impl<T: Timestamp> PerOperatorState<T> {
logging: None,

shared_progress: Rc::new(RefCell::new(SharedProgress::new(inputs,outputs))),
local_caps: Some(ChangeBatch::new()),
internal_summary: Vec::new(),
}
}
Expand All @@ -677,6 +696,8 @@ impl<T: Timestamp> PerOperatorState<T> {
let outputs = scope.outputs();
let notify = scope.notify_me().to_vec();

let local_caps = notify.contains(&FrontierInterest::UpToCapability).then(ChangeBatch::new);

let (internal_summary, shared_progress, operator) = scope.initialize();

if let Some(l) = summary_logging {
Expand Down Expand Up @@ -713,6 +734,7 @@ impl<T: Timestamp> PerOperatorState<T> {
logging,

shared_progress,
local_caps,
internal_summary,
}
}
Expand Down Expand Up @@ -775,7 +797,7 @@ impl<T: Timestamp> PerOperatorState<T> {
/// `outer_pushed` holds the enclosing scope's own output marks: a marked output wired to the
/// scope's outputs (node zero) marks them in turn, so that records leaving nested scopes
/// reach their recipients in the same pass however many scope boundaries they cross.
fn extract_progress(&self, pointstamps: &mut ChangeBatch<(Location, T)>, temp_active: &mut BinaryHeap<Reverse<usize>>, outer_pushed: &mut [bool]) {
fn extract_progress(&mut self, pointstamps: &mut ChangeBatch<(Location, T)>, temp_active: &mut BinaryHeap<Reverse<usize>>, outer_pushed: &mut [bool]) {

let shared_progress = &mut *self.shared_progress.borrow_mut();

Expand All @@ -789,6 +811,7 @@ impl<T: Timestamp> PerOperatorState<T> {
for (output, internal) in shared_progress.internals.iter_mut().enumerate() {
let source = Location::new_source(self.index, output);
for (time, delta) in internal.drain() {
if let Some(caps) = self.local_caps.as_mut() { caps.update(time.clone(), delta); }
pointstamps.update((source, time), delta);
}
}
Expand Down
92 changes: 92 additions & 0 deletions timely/tests/capability_interest.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
//! A region is woken by frontier changes only at or below the capabilities it holds.

use std::cell::RefCell;
use std::collections::HashMap;
use std::rc::Rc;

use timely::dataflow::channels::pact::Pipeline;
use timely::dataflow::operators::generic::builder_rc::OperatorBuilder;
use timely::dataflow::operators::generic::OutputBuilder;
use timely::dataflow::operators::{Enter, Input, Leave, Probe};
use timely::logging::{TimelyEvent, TimelyEventBuilder, StartStop};
use timely::progress::operate::FrontierInterest;

/// Advances an input through `rounds` rounds without data, while an operator in a region holds a
/// capability at `hold_at` and declares `interest`, and returns how often the region was scheduled.
fn region_schedules(hold_at: u64, interest: FrontierInterest, rounds: u64) -> usize {
timely::execute_directly(move |worker| {
let names: Rc<RefCell<HashMap<usize, String>>> = Default::default();
let starts: Rc<RefCell<HashMap<usize, usize>>> = Default::default();
let (names2, starts2) = (Rc::clone(&names), Rc::clone(&starts));
worker.log_register().unwrap().insert::<TimelyEventBuilder, _>("timely", move |_time, data| {
for (_, event) in data.iter().flat_map(|d| d.iter()) {
match event {
TimelyEvent::Operates(o) => { names2.borrow_mut().insert(o.id, o.name.clone()); }
TimelyEvent::Schedule(s) if s.start_stop == StartStop::Start => { *starts2.borrow_mut().entry(s.id).or_default() += 1; }
_ => {}
}
}
});

let held = Rc::new(RefCell::new(None));
let held2 = Rc::clone(&held);
let (mut input, probe) = worker.dataflow::<u64, _, _>(|scope| {
let (input, stream) = scope.new_input::<Vec<u64>>();
let (probe, stream) = stream.probe();
scope.region(|inner| {
let mut builder = OperatorBuilder::new("Hold".to_owned(), inner);
let address = builder.operator_info().address;
let mut input = builder.new_input(stream.enter(inner), Pipeline);
let (output, output_stream) = builder.new_output::<Vec<u64>>();
let mut output = OutputBuilder::from(output);
builder.set_notify_for(0, interest);
builder.build(move |mut capabilities| {
let capability = capabilities.pop().unwrap().delayed(&hold_at);
*held2.borrow_mut() = Some((capability, inner.activator_for(address)));
move |_frontiers| {
let mut output = output.activate();
input.for_each(|time, data| { output.session(&time).give_container(data); });
}
});
output_stream.leave(scope);
});
(input, probe)
});

for round in 1 ..= rounds {
input.advance_to(round);
while probe.less_than(&round) { worker.step(); }
}
let region = names.borrow().iter().find(|(_, n)| *n == "Region").map(|(id, _)| *id).expect("region logged");
let count = starts.borrow().get(&region).copied().unwrap_or(0);

let (capability, activator) = held.borrow_mut().take().unwrap();
drop(capability);
activator.activate();
drop(input);
while worker.step() { }
worker.log_register().unwrap().remove("timely");
count
})
}

#[test]
fn passed_capability_lets_region_sleep() {
let rounds = 100;
let count = region_schedules(0, FrontierInterest::Never, rounds);
assert!(count < 10, "a region holding a passed capability should sleep as frontiers move: {count}");
}

#[test]
fn capability_ahead_wakes_region() {
let rounds = 100;
let count = region_schedules(rounds + 1, FrontierInterest::Never, rounds);
assert!(count >= rounds as usize, "a region holding a capability ahead of the frontier should wake each round: {count}");
}

#[test]
fn if_capability_child_wakes_region() {
let rounds = 100;
let count = region_schedules(0, FrontierInterest::IfCapability, rounds);
assert!(count >= rounds as usize, "an `IfCapability` child should wake its region each round: {count}");
}
83 changes: 83 additions & 0 deletions timely/tests/capability_interest_workers.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
//! An `UpToCapability` operator is woken when the frontier passes a capability it holds, even
//! while another worker's instance of it still holds an earlier capability.

use std::cell::RefCell;
use std::rc::Rc;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};

use timely::dataflow::channels::pact::Pipeline;
use timely::dataflow::operators::generic::builder_rc::OperatorBuilder;
use timely::dataflow::operators::generic::OutputBuilder;
use timely::dataflow::operators::{Input, Probe};
use timely::progress::operate::FrontierInterest;

#[test]
fn wakes_on_own_capability_ahead_of_peers() {
// Worker 0 holds its capability at 5 from its first activation. Worker 1 holds its capability
// at 0 until `release` is set. The input frontier moves 0 -> 3 -> 10, so the move from 3 to 10
// passes worker 0's capability while the capabilities summed over workers still show 0.
let release = Arc::new(AtomicBool::new(false));
let results = timely::execute(timely::Config::process(2), move |worker| {
let index = worker.index();
let release = Arc::clone(&release);
let activator = Rc::new(RefCell::new(None));
let activator2 = Rc::clone(&activator);
// Shared so the test can drop a stalled capability and let the dataflow shut down.
let held = Rc::new(RefCell::new(None));
let held2 = Rc::clone(&held);
let (mut input, input_probe, probe) = worker.dataflow::<u64, _, _>(|scope| {
let (input, stream) = scope.new_input::<Vec<u64>>();
let (input_probe, stream) = stream.probe();
let mut builder = OperatorBuilder::new("Hold".to_owned(), scope);
let address = builder.operator_info().address;
let mut input_handle = builder.new_input(stream, Pipeline);
let (output, output_stream) = builder.new_output::<Vec<u64>>();
let mut output = OutputBuilder::from(output);
builder.set_notify_for(0, FrontierInterest::UpToCapability);
*activator2.borrow_mut() = Some(scope.activator_for(address));
let release = Arc::clone(&release);
builder.build(move |mut capabilities| {
*held2.borrow_mut() = capabilities.pop();
move |frontiers| {
let mut output = output.activate();
input_handle.for_each(|time, data| { output.session(&time).give_container(data); });
let mut capability = held2.borrow_mut();
if let Some(cap) = capability.as_mut() {
if *cap.time() == 0 && (index == 0 || release.load(Ordering::SeqCst)) {
cap.downgrade(&5);
}
}
if capability.as_ref().is_some_and(|cap| *cap.time() == 5 && !frontiers[0].less_equal(&5)) {
*capability = None;
}
}
});
let (probe, _) = output_stream.probe();
(input, input_probe, probe)
});

input.advance_to(3);
while input_probe.less_than(&3) { worker.step(); }
input.advance_to(10);
while input_probe.less_than(&10) { worker.step(); }
for _ in 0 .. 10 { worker.step(); }
// Worker 0 must have seen the frontier pass its capability and released it, before
// worker 1 does anything that could schedule it for another reason.
let released_while_peer_behind = held.borrow().is_none();

if index == 1 { release.store(true, Ordering::SeqCst); }
held.borrow_mut().take();
activator.borrow().as_ref().unwrap().activate();
input.close();
while probe.less_than(&10) { worker.step(); }
(index, released_while_peer_behind)
}).unwrap().join();

for result in results {
let (index, released) = result.unwrap();
if index == 0 {
assert!(released, "worker 0 slept through the frontier passing its capability");
}
}
}
4 changes: 2 additions & 2 deletions timely/tests/frontier_interest_log.rs
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,8 @@ fn interests(frontier: bool) -> Vec<(String, Vec<FrontierInterest>)> {
fn summaries_report_frontier_interest() {
let quiet = interests(false);
assert!(quiet.contains(&("FlatMap".to_string(), vec![FrontierInterest::Never])), "{quiet:?}");
// A region is at least `IfCapability`: it may hold capabilities for messages it has yet to acknowledge.
assert!(quiet.contains(&("Region".to_string(), vec![FrontierInterest::IfCapability])), "{quiet:?}");
// A region is at least `UpToCapability`: it may hold capabilities for messages it has yet to acknowledge.
assert!(quiet.contains(&("Region".to_string(), vec![FrontierInterest::UpToCapability])), "{quiet:?}");
let awake = interests(true);
assert!(awake.contains(&("Inspect".to_string(), vec![FrontierInterest::Always])), "{awake:?}");
assert!(awake.contains(&("Region".to_string(), vec![FrontierInterest::Always])), "{awake:?}");
Expand Down