From bac426a69bc0748678481dad6affbd1133a1503d Mon Sep 17 00:00:00 2001 From: Dov Alperin Date: Mon, 16 Nov 2026 15:17:38 -0500 Subject: [PATCH 1/2] Add FrontierInterest::UpToCapability and use it for regions An operator declaring `UpToCapability` is woken by a frontier change only when the change removes an element less or equal to a capability it holds. Changes entirely above its capabilities, including all changes once the frontier has passed every capability it holds, do not schedule it. A subgraph's child zero now declares `UpToCapability`, so a region whose children are all `Never` or `UpToCapability` sleeps through frontier moves that cannot affect the capabilities it holds. A child declaring `IfCapability` or `Always` still raises the region to that interest. --- timely/src/progress/operate.rs | 13 +++- timely/src/progress/subgraph.rs | 10 ++- timely/tests/capability_interest.rs | 92 +++++++++++++++++++++++++++ timely/tests/frontier_interest_log.rs | 4 +- 4 files changed, 115 insertions(+), 4 deletions(-) create mode 100644 timely/tests/capability_interest.rs diff --git a/timely/src/progress/operate.rs b/timely/src/progress/operate.rs index ca352ff56..9211e4c6a 100644 --- a/timely/src/progress/operate.rs +++ b/timely/src/progress/operate.rs @@ -64,11 +64,22 @@ pub trait Operate { } /// The ways in which an operator can express interest in activation when an input frontier changes. +/// +/// Variants are ordered from least to most interested. #[derive(serde::Serialize, serde::Deserialize, columnar::Columnar, Ord, PartialOrd, Eq, PartialEq, Copy, Clone, Debug)] pub enum FrontierInterest { /// Never interested in frontier changes, as for example the `map()` and `filter()` operators. Never, - /// Interested when the operator holds capabilities. + /// Interested when a frontier change removes an element less or equal to a held capability. + /// + /// Such a change is the only kind that can complete the time of a held capability, or a time + /// greater than one. The operator is not woken for changes entirely above its capabilities, + /// which includes all changes once the frontier has passed every capability it holds. + /// Any time the operator waits on must be greater or equal to a held capability that the + /// frontier has not yet passed; a capability the frontier has passed does not keep the + /// operator awake for later times. + UpToCapability, + /// Interested in any frontier change while the operator holds capabilities. IfCapability, /// Always interested in frontier changes, as for example the `probe()` and `capture()` operators. Always, diff --git a/timely/src/progress/subgraph.rs b/timely/src/progress/subgraph.rs index ee014452d..5e7e185a7 100644 --- a/timely/src/progress/subgraph.rs +++ b/timely/src/progress/subgraph.rs @@ -194,6 +194,9 @@ where activations.borrow_mut().activate(&self.path[..]); // The subgraph's per-input interest is conservatively the max across all children's inputs. + // A child's `UpToCapability` interest is covered by the subgraph's own: path summaries do not + // decrease times, so a frontier change at or below a child's capability starts from a removed + // scope-input element at or below that capability's image at the scope outputs. let max_interest = children.iter() .flat_map(|c| c.notify.iter().copied()) .max() @@ -501,6 +504,11 @@ where let activate = match self.children[location.node].notify[port] { FrontierInterest::Always => true, FrontierInterest::IfCapability => { operators[location.node].cap_counts > 0 } + FrontierInterest::UpToCapability => { + diff < 0 && operators[location.node].sources.iter().any(|source| { + source.pointstamps.frontier().iter().any(|cap| time.less_equal(cap)) + }) + } FrontierInterest::Never => false, }; if activate { self.temp_active.push(Reverse(location.node)); } @@ -651,7 +659,7 @@ impl PerOperatorState { index: 0, id: usize::MAX, local: false, - notify: vec![FrontierInterest::IfCapability; inputs], + notify: vec![FrontierInterest::UpToCapability; inputs], inputs, outputs, diff --git a/timely/tests/capability_interest.rs b/timely/tests/capability_interest.rs new file mode 100644 index 000000000..071690d7c --- /dev/null +++ b/timely/tests/capability_interest.rs @@ -0,0 +1,92 @@ +//! A region is woken by frontier changes only at or below the capabilities it holds. + +use std::cell::RefCell; +use std::collections::HashMap; +use std::rc::Rc; + +use timely::dataflow::channels::pact::Pipeline; +use timely::dataflow::operators::generic::builder_rc::OperatorBuilder; +use timely::dataflow::operators::generic::OutputBuilder; +use timely::dataflow::operators::{Enter, Input, Leave, Probe}; +use timely::logging::{TimelyEvent, TimelyEventBuilder, StartStop}; +use timely::progress::operate::FrontierInterest; + +/// Advances an input through `rounds` rounds without data, while an operator in a region holds a +/// capability at `hold_at` and declares `interest`, and returns how often the region was scheduled. +fn region_schedules(hold_at: u64, interest: FrontierInterest, rounds: u64) -> usize { + timely::execute_directly(move |worker| { + let names: Rc>> = Default::default(); + let starts: Rc>> = Default::default(); + let (names2, starts2) = (Rc::clone(&names), Rc::clone(&starts)); + worker.log_register().unwrap().insert::("timely", move |_time, data| { + for (_, event) in data.iter().flat_map(|d| d.iter()) { + match event { + TimelyEvent::Operates(o) => { names2.borrow_mut().insert(o.id, o.name.clone()); } + TimelyEvent::Schedule(s) if s.start_stop == StartStop::Start => { *starts2.borrow_mut().entry(s.id).or_default() += 1; } + _ => {} + } + } + }); + + let held = Rc::new(RefCell::new(None)); + let held2 = Rc::clone(&held); + let (mut input, probe) = worker.dataflow::(|scope| { + let (input, stream) = scope.new_input::>(); + let (probe, stream) = stream.probe(); + scope.region(|inner| { + let mut builder = OperatorBuilder::new("Hold".to_owned(), inner); + let address = builder.operator_info().address; + let mut input = builder.new_input(stream.enter(inner), Pipeline); + let (output, output_stream) = builder.new_output::>(); + let mut output = OutputBuilder::from(output); + builder.set_notify_for(0, interest); + builder.build(move |mut capabilities| { + let capability = capabilities.pop().unwrap().delayed(&hold_at); + *held2.borrow_mut() = Some((capability, inner.activator_for(address))); + move |_frontiers| { + let mut output = output.activate(); + input.for_each(|time, data| { output.session(&time).give_container(data); }); + } + }); + output_stream.leave(scope); + }); + (input, probe) + }); + + for round in 1 ..= rounds { + input.advance_to(round); + while probe.less_than(&round) { worker.step(); } + } + let region = names.borrow().iter().find(|(_, n)| *n == "Region").map(|(id, _)| *id).expect("region logged"); + let count = starts.borrow().get(®ion).copied().unwrap_or(0); + + let (capability, activator) = held.borrow_mut().take().unwrap(); + drop(capability); + activator.activate(); + drop(input); + while worker.step() { } + worker.log_register().unwrap().remove("timely"); + count + }) +} + +#[test] +fn passed_capability_lets_region_sleep() { + let rounds = 100; + let count = region_schedules(0, FrontierInterest::Never, rounds); + assert!(count < 10, "a region holding a passed capability should sleep as frontiers move: {count}"); +} + +#[test] +fn capability_ahead_wakes_region() { + let rounds = 100; + let count = region_schedules(rounds + 1, FrontierInterest::Never, rounds); + assert!(count >= rounds as usize, "a region holding a capability ahead of the frontier should wake each round: {count}"); +} + +#[test] +fn if_capability_child_wakes_region() { + let rounds = 100; + let count = region_schedules(0, FrontierInterest::IfCapability, rounds); + assert!(count >= rounds as usize, "an `IfCapability` child should wake its region each round: {count}"); +} diff --git a/timely/tests/frontier_interest_log.rs b/timely/tests/frontier_interest_log.rs index 5caedad2a..356f55f18 100644 --- a/timely/tests/frontier_interest_log.rs +++ b/timely/tests/frontier_interest_log.rs @@ -54,8 +54,8 @@ fn interests(frontier: bool) -> Vec<(String, Vec)> { fn summaries_report_frontier_interest() { let quiet = interests(false); assert!(quiet.contains(&("FlatMap".to_string(), vec![FrontierInterest::Never])), "{quiet:?}"); - // A region is at least `IfCapability`: it may hold capabilities for messages it has yet to acknowledge. - assert!(quiet.contains(&("Region".to_string(), vec![FrontierInterest::IfCapability])), "{quiet:?}"); + // A region is at least `UpToCapability`: it may hold capabilities for messages it has yet to acknowledge. + assert!(quiet.contains(&("Region".to_string(), vec![FrontierInterest::UpToCapability])), "{quiet:?}"); let awake = interests(true); assert!(awake.contains(&("Inspect".to_string(), vec![FrontierInterest::Always])), "{awake:?}"); assert!(awake.contains(&("Region".to_string(), vec![FrontierInterest::Always])), "{awake:?}"); From 335a6943d0b26e36a5aea105370722427a1bd034 Mon Sep 17 00:00:00 2001 From: Dov Alperin Date: Mon, 16 Nov 2026 17:31:49 -0500 Subject: [PATCH 2/2] Judge UpToCapability against this worker's capabilities The wake check compared a removed frontier element against the tracker's source pointstamps. Those sum the operator's capabilities over all workers as of the last progress exchange, so a capability held by this worker's instance can sit above every element they show. The frontier then passes it without waking the operator, and the other workers' later downgrades change only capabilities, never the input frontier, so nothing re-checks. Each child with an `UpToCapability` input now keeps a tally of the capabilities it holds on this worker, summed over its outputs and updated as `extract_progress` drains its internals. The check uses that tally. A region's tally is the capabilities it reports, as before. --- timely/src/progress/operate.rs | 2 + timely/src/progress/subgraph.rs | 23 +++++- timely/tests/capability_interest_workers.rs | 83 +++++++++++++++++++++ 3 files changed, 104 insertions(+), 4 deletions(-) create mode 100644 timely/tests/capability_interest_workers.rs diff --git a/timely/src/progress/operate.rs b/timely/src/progress/operate.rs index 9211e4c6a..d882229f7 100644 --- a/timely/src/progress/operate.rs +++ b/timely/src/progress/operate.rs @@ -71,6 +71,8 @@ pub enum FrontierInterest { /// Never interested in frontier changes, as for example the `map()` and `filter()` operators. Never, /// Interested when a frontier change removes an element less or equal to a held capability. + /// Held capabilities are those of this worker's instance of the operator, not those of its + /// instances on other workers. /// /// Such a change is the only kind that can complete the time of a held capability, or a time /// greater than one. The operator is not woken for changes entirely above its capabilities, diff --git a/timely/src/progress/subgraph.rs b/timely/src/progress/subgraph.rs index 5e7e185a7..aae862349 100644 --- a/timely/src/progress/subgraph.rs +++ b/timely/src/progress/subgraph.rs @@ -501,12 +501,18 @@ where // Targets are actionable, sources are not. if let crate::progress::Port::Target(port) = location.port { // Activate based on expressed frontier interest for this input. - let activate = match self.children[location.node].notify[port] { + let child = &mut self.children[location.node]; + let activate = match child.notify[port] { FrontierInterest::Always => true, FrontierInterest::IfCapability => { operators[location.node].cap_counts > 0 } + // Judged against the capabilities this worker's instance holds. The tracker's + // pointstamps sum all workers and trail this worker's own downgrades, so a + // capability held here can sit above every element they show; the other + // workers' downgrades arrive later as capability changes, which change no + // frontier and so would not revisit a missed wake. FrontierInterest::UpToCapability => { - diff < 0 && operators[location.node].sources.iter().any(|source| { - source.pointstamps.frontier().iter().any(|cap| time.less_equal(cap)) + diff < 0 && child.local_caps.as_mut().is_some_and(|caps| { + caps.iter().any(|(cap, count)| *count > 0 && time.less_equal(cap)) }) } FrontierInterest::Never => false, @@ -645,6 +651,10 @@ struct PerOperatorState { shared_progress: Rc>>, + /// Capabilities this worker's instance holds, summed over its outputs and accumulated from its + /// internals. Kept only for operators with an `UpToCapability` input. + local_caps: Option>, + internal_summary: Connectivity, // from initialize; moved into the reachability builder. logging: Option, @@ -668,6 +678,7 @@ impl PerOperatorState { logging: None, shared_progress: Rc::new(RefCell::new(SharedProgress::new(inputs,outputs))), + local_caps: Some(ChangeBatch::new()), internal_summary: Vec::new(), } } @@ -685,6 +696,8 @@ impl PerOperatorState { let outputs = scope.outputs(); let notify = scope.notify_me().to_vec(); + let local_caps = notify.contains(&FrontierInterest::UpToCapability).then(ChangeBatch::new); + let (internal_summary, shared_progress, operator) = scope.initialize(); if let Some(l) = summary_logging { @@ -721,6 +734,7 @@ impl PerOperatorState { logging, shared_progress, + local_caps, internal_summary, } } @@ -783,7 +797,7 @@ impl PerOperatorState { /// `outer_pushed` holds the enclosing scope's own output marks: a marked output wired to the /// scope's outputs (node zero) marks them in turn, so that records leaving nested scopes /// reach their recipients in the same pass however many scope boundaries they cross. - fn extract_progress(&self, pointstamps: &mut ChangeBatch<(Location, T)>, temp_active: &mut BinaryHeap>, outer_pushed: &mut [bool]) { + fn extract_progress(&mut self, pointstamps: &mut ChangeBatch<(Location, T)>, temp_active: &mut BinaryHeap>, outer_pushed: &mut [bool]) { let shared_progress = &mut *self.shared_progress.borrow_mut(); @@ -797,6 +811,7 @@ impl PerOperatorState { for (output, internal) in shared_progress.internals.iter_mut().enumerate() { let source = Location::new_source(self.index, output); for (time, delta) in internal.drain() { + if let Some(caps) = self.local_caps.as_mut() { caps.update(time.clone(), delta); } pointstamps.update((source, time), delta); } } diff --git a/timely/tests/capability_interest_workers.rs b/timely/tests/capability_interest_workers.rs new file mode 100644 index 000000000..f8a479636 --- /dev/null +++ b/timely/tests/capability_interest_workers.rs @@ -0,0 +1,83 @@ +//! An `UpToCapability` operator is woken when the frontier passes a capability it holds, even +//! while another worker's instance of it still holds an earlier capability. + +use std::cell::RefCell; +use std::rc::Rc; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; + +use timely::dataflow::channels::pact::Pipeline; +use timely::dataflow::operators::generic::builder_rc::OperatorBuilder; +use timely::dataflow::operators::generic::OutputBuilder; +use timely::dataflow::operators::{Input, Probe}; +use timely::progress::operate::FrontierInterest; + +#[test] +fn wakes_on_own_capability_ahead_of_peers() { + // Worker 0 holds its capability at 5 from its first activation. Worker 1 holds its capability + // at 0 until `release` is set. The input frontier moves 0 -> 3 -> 10, so the move from 3 to 10 + // passes worker 0's capability while the capabilities summed over workers still show 0. + let release = Arc::new(AtomicBool::new(false)); + let results = timely::execute(timely::Config::process(2), move |worker| { + let index = worker.index(); + let release = Arc::clone(&release); + let activator = Rc::new(RefCell::new(None)); + let activator2 = Rc::clone(&activator); + // Shared so the test can drop a stalled capability and let the dataflow shut down. + let held = Rc::new(RefCell::new(None)); + let held2 = Rc::clone(&held); + let (mut input, input_probe, probe) = worker.dataflow::(|scope| { + let (input, stream) = scope.new_input::>(); + let (input_probe, stream) = stream.probe(); + let mut builder = OperatorBuilder::new("Hold".to_owned(), scope); + let address = builder.operator_info().address; + let mut input_handle = builder.new_input(stream, Pipeline); + let (output, output_stream) = builder.new_output::>(); + let mut output = OutputBuilder::from(output); + builder.set_notify_for(0, FrontierInterest::UpToCapability); + *activator2.borrow_mut() = Some(scope.activator_for(address)); + let release = Arc::clone(&release); + builder.build(move |mut capabilities| { + *held2.borrow_mut() = capabilities.pop(); + move |frontiers| { + let mut output = output.activate(); + input_handle.for_each(|time, data| { output.session(&time).give_container(data); }); + let mut capability = held2.borrow_mut(); + if let Some(cap) = capability.as_mut() { + if *cap.time() == 0 && (index == 0 || release.load(Ordering::SeqCst)) { + cap.downgrade(&5); + } + } + if capability.as_ref().is_some_and(|cap| *cap.time() == 5 && !frontiers[0].less_equal(&5)) { + *capability = None; + } + } + }); + let (probe, _) = output_stream.probe(); + (input, input_probe, probe) + }); + + input.advance_to(3); + while input_probe.less_than(&3) { worker.step(); } + input.advance_to(10); + while input_probe.less_than(&10) { worker.step(); } + for _ in 0 .. 10 { worker.step(); } + // Worker 0 must have seen the frontier pass its capability and released it, before + // worker 1 does anything that could schedule it for another reason. + let released_while_peer_behind = held.borrow().is_none(); + + if index == 1 { release.store(true, Ordering::SeqCst); } + held.borrow_mut().take(); + activator.borrow().as_ref().unwrap().activate(); + input.close(); + while probe.less_than(&10) { worker.step(); } + (index, released_while_peer_behind) + }).unwrap().join(); + + for result in results { + let (index, released) = result.unwrap(); + if index == 0 { + assert!(released, "worker 0 slept through the frontier passing its capability"); + } + } +}