diff --git a/timely/src/progress/operate.rs b/timely/src/progress/operate.rs index ca352ff56..d882229f7 100644 --- a/timely/src/progress/operate.rs +++ b/timely/src/progress/operate.rs @@ -64,11 +64,24 @@ pub trait Operate { } /// The ways in which an operator can express interest in activation when an input frontier changes. +/// +/// Variants are ordered from least to most interested. #[derive(serde::Serialize, serde::Deserialize, columnar::Columnar, Ord, PartialOrd, Eq, PartialEq, Copy, Clone, Debug)] pub enum FrontierInterest { /// Never interested in frontier changes, as for example the `map()` and `filter()` operators. Never, - /// Interested when the operator holds capabilities. + /// Interested when a frontier change removes an element less or equal to a held capability. + /// Held capabilities are those of this worker's instance of the operator, not those of its + /// instances on other workers. + /// + /// Such a change is the only kind that can complete the time of a held capability, or a time + /// greater than one. The operator is not woken for changes entirely above its capabilities, + /// which includes all changes once the frontier has passed every capability it holds. + /// Any time the operator waits on must be greater or equal to a held capability that the + /// frontier has not yet passed; a capability the frontier has passed does not keep the + /// operator awake for later times. + UpToCapability, + /// Interested in any frontier change while the operator holds capabilities. IfCapability, /// Always interested in frontier changes, as for example the `probe()` and `capture()` operators. Always, diff --git a/timely/src/progress/subgraph.rs b/timely/src/progress/subgraph.rs index ee014452d..aae862349 100644 --- a/timely/src/progress/subgraph.rs +++ b/timely/src/progress/subgraph.rs @@ -194,6 +194,9 @@ where activations.borrow_mut().activate(&self.path[..]); // The subgraph's per-input interest is conservatively the max across all children's inputs. + // A child's `UpToCapability` interest is covered by the subgraph's own: path summaries do not + // decrease times, so a frontier change at or below a child's capability starts from a removed + // scope-input element at or below that capability's image at the scope outputs. let max_interest = children.iter() .flat_map(|c| c.notify.iter().copied()) .max() @@ -498,9 +501,20 @@ where // Targets are actionable, sources are not. if let crate::progress::Port::Target(port) = location.port { // Activate based on expressed frontier interest for this input. - let activate = match self.children[location.node].notify[port] { + let child = &mut self.children[location.node]; + let activate = match child.notify[port] { FrontierInterest::Always => true, FrontierInterest::IfCapability => { operators[location.node].cap_counts > 0 } + // Judged against the capabilities this worker's instance holds. The tracker's + // pointstamps sum all workers and trail this worker's own downgrades, so a + // capability held here can sit above every element they show; the other + // workers' downgrades arrive later as capability changes, which change no + // frontier and so would not revisit a missed wake. + FrontierInterest::UpToCapability => { + diff < 0 && child.local_caps.as_mut().is_some_and(|caps| { + caps.iter().any(|(cap, count)| *count > 0 && time.less_equal(cap)) + }) + } FrontierInterest::Never => false, }; if activate { self.temp_active.push(Reverse(location.node)); } @@ -637,6 +651,10 @@ struct PerOperatorState { shared_progress: Rc>>, + /// Capabilities this worker's instance holds, summed over its outputs and accumulated from its + /// internals. Kept only for operators with an `UpToCapability` input. + local_caps: Option>, + internal_summary: Connectivity, // from initialize; moved into the reachability builder. logging: Option, @@ -651,7 +669,7 @@ impl PerOperatorState { index: 0, id: usize::MAX, local: false, - notify: vec![FrontierInterest::IfCapability; inputs], + notify: vec![FrontierInterest::UpToCapability; inputs], inputs, outputs, @@ -660,6 +678,7 @@ impl PerOperatorState { logging: None, shared_progress: Rc::new(RefCell::new(SharedProgress::new(inputs,outputs))), + local_caps: Some(ChangeBatch::new()), internal_summary: Vec::new(), } } @@ -677,6 +696,8 @@ impl PerOperatorState { let outputs = scope.outputs(); let notify = scope.notify_me().to_vec(); + let local_caps = notify.contains(&FrontierInterest::UpToCapability).then(ChangeBatch::new); + let (internal_summary, shared_progress, operator) = scope.initialize(); if let Some(l) = summary_logging { @@ -713,6 +734,7 @@ impl PerOperatorState { logging, shared_progress, + local_caps, internal_summary, } } @@ -775,7 +797,7 @@ impl PerOperatorState { /// `outer_pushed` holds the enclosing scope's own output marks: a marked output wired to the /// scope's outputs (node zero) marks them in turn, so that records leaving nested scopes /// reach their recipients in the same pass however many scope boundaries they cross. - fn extract_progress(&self, pointstamps: &mut ChangeBatch<(Location, T)>, temp_active: &mut BinaryHeap>, outer_pushed: &mut [bool]) { + fn extract_progress(&mut self, pointstamps: &mut ChangeBatch<(Location, T)>, temp_active: &mut BinaryHeap>, outer_pushed: &mut [bool]) { let shared_progress = &mut *self.shared_progress.borrow_mut(); @@ -789,6 +811,7 @@ impl PerOperatorState { for (output, internal) in shared_progress.internals.iter_mut().enumerate() { let source = Location::new_source(self.index, output); for (time, delta) in internal.drain() { + if let Some(caps) = self.local_caps.as_mut() { caps.update(time.clone(), delta); } pointstamps.update((source, time), delta); } } diff --git a/timely/tests/capability_interest.rs b/timely/tests/capability_interest.rs new file mode 100644 index 000000000..071690d7c --- /dev/null +++ b/timely/tests/capability_interest.rs @@ -0,0 +1,92 @@ +//! A region is woken by frontier changes only at or below the capabilities it holds. + +use std::cell::RefCell; +use std::collections::HashMap; +use std::rc::Rc; + +use timely::dataflow::channels::pact::Pipeline; +use timely::dataflow::operators::generic::builder_rc::OperatorBuilder; +use timely::dataflow::operators::generic::OutputBuilder; +use timely::dataflow::operators::{Enter, Input, Leave, Probe}; +use timely::logging::{TimelyEvent, TimelyEventBuilder, StartStop}; +use timely::progress::operate::FrontierInterest; + +/// Advances an input through `rounds` rounds without data, while an operator in a region holds a +/// capability at `hold_at` and declares `interest`, and returns how often the region was scheduled. +fn region_schedules(hold_at: u64, interest: FrontierInterest, rounds: u64) -> usize { + timely::execute_directly(move |worker| { + let names: Rc>> = Default::default(); + let starts: Rc>> = Default::default(); + let (names2, starts2) = (Rc::clone(&names), Rc::clone(&starts)); + worker.log_register().unwrap().insert::("timely", move |_time, data| { + for (_, event) in data.iter().flat_map(|d| d.iter()) { + match event { + TimelyEvent::Operates(o) => { names2.borrow_mut().insert(o.id, o.name.clone()); } + TimelyEvent::Schedule(s) if s.start_stop == StartStop::Start => { *starts2.borrow_mut().entry(s.id).or_default() += 1; } + _ => {} + } + } + }); + + let held = Rc::new(RefCell::new(None)); + let held2 = Rc::clone(&held); + let (mut input, probe) = worker.dataflow::(|scope| { + let (input, stream) = scope.new_input::>(); + let (probe, stream) = stream.probe(); + scope.region(|inner| { + let mut builder = OperatorBuilder::new("Hold".to_owned(), inner); + let address = builder.operator_info().address; + let mut input = builder.new_input(stream.enter(inner), Pipeline); + let (output, output_stream) = builder.new_output::>(); + let mut output = OutputBuilder::from(output); + builder.set_notify_for(0, interest); + builder.build(move |mut capabilities| { + let capability = capabilities.pop().unwrap().delayed(&hold_at); + *held2.borrow_mut() = Some((capability, inner.activator_for(address))); + move |_frontiers| { + let mut output = output.activate(); + input.for_each(|time, data| { output.session(&time).give_container(data); }); + } + }); + output_stream.leave(scope); + }); + (input, probe) + }); + + for round in 1 ..= rounds { + input.advance_to(round); + while probe.less_than(&round) { worker.step(); } + } + let region = names.borrow().iter().find(|(_, n)| *n == "Region").map(|(id, _)| *id).expect("region logged"); + let count = starts.borrow().get(®ion).copied().unwrap_or(0); + + let (capability, activator) = held.borrow_mut().take().unwrap(); + drop(capability); + activator.activate(); + drop(input); + while worker.step() { } + worker.log_register().unwrap().remove("timely"); + count + }) +} + +#[test] +fn passed_capability_lets_region_sleep() { + let rounds = 100; + let count = region_schedules(0, FrontierInterest::Never, rounds); + assert!(count < 10, "a region holding a passed capability should sleep as frontiers move: {count}"); +} + +#[test] +fn capability_ahead_wakes_region() { + let rounds = 100; + let count = region_schedules(rounds + 1, FrontierInterest::Never, rounds); + assert!(count >= rounds as usize, "a region holding a capability ahead of the frontier should wake each round: {count}"); +} + +#[test] +fn if_capability_child_wakes_region() { + let rounds = 100; + let count = region_schedules(0, FrontierInterest::IfCapability, rounds); + assert!(count >= rounds as usize, "an `IfCapability` child should wake its region each round: {count}"); +} diff --git a/timely/tests/capability_interest_workers.rs b/timely/tests/capability_interest_workers.rs new file mode 100644 index 000000000..f8a479636 --- /dev/null +++ b/timely/tests/capability_interest_workers.rs @@ -0,0 +1,83 @@ +//! An `UpToCapability` operator is woken when the frontier passes a capability it holds, even +//! while another worker's instance of it still holds an earlier capability. + +use std::cell::RefCell; +use std::rc::Rc; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; + +use timely::dataflow::channels::pact::Pipeline; +use timely::dataflow::operators::generic::builder_rc::OperatorBuilder; +use timely::dataflow::operators::generic::OutputBuilder; +use timely::dataflow::operators::{Input, Probe}; +use timely::progress::operate::FrontierInterest; + +#[test] +fn wakes_on_own_capability_ahead_of_peers() { + // Worker 0 holds its capability at 5 from its first activation. Worker 1 holds its capability + // at 0 until `release` is set. The input frontier moves 0 -> 3 -> 10, so the move from 3 to 10 + // passes worker 0's capability while the capabilities summed over workers still show 0. + let release = Arc::new(AtomicBool::new(false)); + let results = timely::execute(timely::Config::process(2), move |worker| { + let index = worker.index(); + let release = Arc::clone(&release); + let activator = Rc::new(RefCell::new(None)); + let activator2 = Rc::clone(&activator); + // Shared so the test can drop a stalled capability and let the dataflow shut down. + let held = Rc::new(RefCell::new(None)); + let held2 = Rc::clone(&held); + let (mut input, input_probe, probe) = worker.dataflow::(|scope| { + let (input, stream) = scope.new_input::>(); + let (input_probe, stream) = stream.probe(); + let mut builder = OperatorBuilder::new("Hold".to_owned(), scope); + let address = builder.operator_info().address; + let mut input_handle = builder.new_input(stream, Pipeline); + let (output, output_stream) = builder.new_output::>(); + let mut output = OutputBuilder::from(output); + builder.set_notify_for(0, FrontierInterest::UpToCapability); + *activator2.borrow_mut() = Some(scope.activator_for(address)); + let release = Arc::clone(&release); + builder.build(move |mut capabilities| { + *held2.borrow_mut() = capabilities.pop(); + move |frontiers| { + let mut output = output.activate(); + input_handle.for_each(|time, data| { output.session(&time).give_container(data); }); + let mut capability = held2.borrow_mut(); + if let Some(cap) = capability.as_mut() { + if *cap.time() == 0 && (index == 0 || release.load(Ordering::SeqCst)) { + cap.downgrade(&5); + } + } + if capability.as_ref().is_some_and(|cap| *cap.time() == 5 && !frontiers[0].less_equal(&5)) { + *capability = None; + } + } + }); + let (probe, _) = output_stream.probe(); + (input, input_probe, probe) + }); + + input.advance_to(3); + while input_probe.less_than(&3) { worker.step(); } + input.advance_to(10); + while input_probe.less_than(&10) { worker.step(); } + for _ in 0 .. 10 { worker.step(); } + // Worker 0 must have seen the frontier pass its capability and released it, before + // worker 1 does anything that could schedule it for another reason. + let released_while_peer_behind = held.borrow().is_none(); + + if index == 1 { release.store(true, Ordering::SeqCst); } + held.borrow_mut().take(); + activator.borrow().as_ref().unwrap().activate(); + input.close(); + while probe.less_than(&10) { worker.step(); } + (index, released_while_peer_behind) + }).unwrap().join(); + + for result in results { + let (index, released) = result.unwrap(); + if index == 0 { + assert!(released, "worker 0 slept through the frontier passing its capability"); + } + } +} diff --git a/timely/tests/frontier_interest_log.rs b/timely/tests/frontier_interest_log.rs index 5caedad2a..356f55f18 100644 --- a/timely/tests/frontier_interest_log.rs +++ b/timely/tests/frontier_interest_log.rs @@ -54,8 +54,8 @@ fn interests(frontier: bool) -> Vec<(String, Vec)> { fn summaries_report_frontier_interest() { let quiet = interests(false); assert!(quiet.contains(&("FlatMap".to_string(), vec![FrontierInterest::Never])), "{quiet:?}"); - // A region is at least `IfCapability`: it may hold capabilities for messages it has yet to acknowledge. - assert!(quiet.contains(&("Region".to_string(), vec![FrontierInterest::IfCapability])), "{quiet:?}"); + // A region is at least `UpToCapability`: it may hold capabilities for messages it has yet to acknowledge. + assert!(quiet.contains(&("Region".to_string(), vec![FrontierInterest::UpToCapability])), "{quiet:?}"); let awake = interests(true); assert!(awake.contains(&("Inspect".to_string(), vec![FrontierInterest::Always])), "{awake:?}"); assert!(awake.contains(&("Region".to_string(), vec![FrontierInterest::Always])), "{awake:?}");