diff --git a/src/IR.cpp b/src/IR.cpp index 72058b62700d..fb1a456cf7e5 100644 --- a/src/IR.cpp +++ b/src/IR.cpp @@ -818,7 +818,10 @@ constexpr const char *intrinsic_op_names[] = { "count_leading_zeros", "count_trailing_zeros", "debug_to_file", + "declare_allocation", + "declare_box_required_at_root", "declare_box_touched", + "declare_stage", "div_round_to_zero", "dynamic_shuffle", "extract_bits", diff --git a/src/IR.h b/src/IR.h index f260890a48e1..93f9828e4171 100644 --- a/src/IR.h +++ b/src/IR.h @@ -725,8 +725,23 @@ struct Call : public ExprNode { count_leading_zeros, count_trailing_zeros, debug_to_file, + // Emitted (when profiling) for a device-only allocation that has no + // host-side Allocate node: the buffer lives solely on the device, so + // InjectHostDevBufferCopies elides its host allocation. Lets the + // profiler, which tracks memory at host Allocate nodes, still see the + // allocation and bill its size to the Func. + // Args: (StringImm Func name, size in bytes, IntImm memory type). + declare_allocation, + // Declares that region required of a particular Func at this + // scope. Injected by ScheduleFunctions and used by the profiler. + declare_box_required_at_root, // Declares that a box region of an allocation has been touched (used by bounds inference) declare_box_touched, + // Declares that the following stmt computes a particular stage of + // a particular Func. Used by the profiler to bill points computed + // in the pure def separately from points computed in update defs. + // Args: (Variable handle for the func, Int<32> stage_idx). + declare_stage, div_round_to_zero, // A shuffle operation with runtime-varying indices. dynamic_shuffle, diff --git a/src/InjectHostDevBufferCopies.cpp b/src/InjectHostDevBufferCopies.cpp index fdbaef642443..1abae28402b7 100644 --- a/src/InjectHostDevBufferCopies.cpp +++ b/src/InjectHostDevBufferCopies.cpp @@ -6,6 +6,7 @@ #include "IRMutator.h" #include "IROperator.h" #include "IRPrinter.h" +#include "Simplify.h" #include "Substitute.h" #include @@ -490,6 +491,12 @@ class InjectBufferCopies : public IRMutator { protected: using IRMutator::visit; + // Whether the pipeline is being profiled, so we know whether to emit + // allocation markers for device-only buffers whose host allocation we + // strip below (the profiler tracks memory at the host Allocate, which + // no longer carries a size for these). + bool profiling; + // Inject the registration of a device destructor just after the // .buffer symbol is defined (which is safely before the first // device_malloc). @@ -658,6 +665,25 @@ class InjectBufferCopies : public IRMutator { // references to it (e.g. the one in the make_buffer // call) with NULL. body = substitute(op->name, reinterpret(Handle(), make_zero(UInt(64))), body); + if (profiling) { + // The storage still exists on the device, but with no + // host Allocate size the profiler can't see it. Emit a + // declare_allocation marker carrying the device + // allocation's byte size so it gets billed to this Func. + // A 0-dimensional allocation (empty extents) holds one + // element, so start the product at the element size. + Expr size_bytes = make_const(UInt(64), op->type.bytes()); + for (const Expr &extent : op->extents) { + size_bytes *= cast(extent); + } + size_bytes = simplify(size_bytes); + Expr marker = Call::make(Int(32), Call::declare_allocation, + {Expr(op->name), + size_bytes, + make_const(Int(32), (int)op->memory_type)}, + Call::Intrinsic); + body = Block::make(Evaluate::make(marker), body); + } } return op->with(op->extents, condition, body); @@ -674,6 +700,11 @@ class InjectBufferCopies : public IRMutator { return IRMutator::visit(op); } } + +public: + InjectBufferCopies(bool profiling) + : profiling(profiling) { + } }; // Find the site in the IR where we want to inject the copies/dirty @@ -794,7 +825,8 @@ Stmt inject_host_dev_buffer_copies(Stmt s, const Target &t) { } // Handle internal allocations - s = InjectBufferCopies()(s); + bool profiling = t.has_feature(Target::Profile) || t.has_feature(Target::ProfileByTimer); + s = InjectBufferCopies(profiling)(s); // Handle inputs and outputs FindOutermostProduce outermost; diff --git a/src/Lower.cpp b/src/Lower.cpp index 409ae847baa3..9730585b51a6 100644 --- a/src/Lower.cpp +++ b/src/Lower.cpp @@ -404,12 +404,6 @@ void lower_impl(const vector &output_funcs, s = bound_small_allocations(s); log("Lowering after bounding small allocations:", s); - if (t.has_feature(Target::Profile) || t.has_feature(Target::ProfileByTimer)) { - debug(1) << "Injecting profiling...\n"; - s = inject_profiling(s, pipeline_name, env); - log("Lowering after injecting profiling:", s); - } - if (t.has_feature(Target::CUDA)) { debug(1) << "Injecting warp shuffles...\n"; s = lower_warp_shuffles(s, t); @@ -440,6 +434,13 @@ void lower_impl(const vector &output_funcs, s = hoist_loop_invariant_if_statements(s); log("Lowering after removing dead allocations and hoisting loop invariants:", s); + if (t.has_feature(Target::Profile) || t.has_feature(Target::ProfileByTimer)) { + debug(1) << "Injecting profiling...\n"; + s = inject_profiling(s, pipeline_name, env, t); + s = simplify(s); + log("Lowering after injecting profiling:", s); + } + debug(1) << "Finding intrinsics...\n"; // Must be run after the last simplification, because it turns // divisions into shifts, which the simplifier reverses. diff --git a/src/Profiling.cpp b/src/Profiling.cpp index 5a87c898787e..3849f8936905 100644 --- a/src/Profiling.cpp +++ b/src/Profiling.cpp @@ -1,9 +1,14 @@ #include +#include #include #include #include +#include "Bounds.h" #include "CodeGen_Internal.h" +#include "DeviceInterface.h" +#include "ExprUsesVar.h" +#include "FindCalls.h" #include "Function.h" #include "IRMutator.h" #include "IROperator.h" @@ -11,6 +16,7 @@ #include "Profiling.h" #include "Scope.h" #include "Simplify.h" +#include "Solve.h" #include "Substitute.h" #include "UniquifyVariableNames.h" #include "Util.h" @@ -70,6 +76,7 @@ struct Names { std::string profiler_func_stack_peak_buf; std::string profiler_func_kinds; std::string profiler_func_buffer_func_ids; + std::string profiler_func_counters_approximated; std::string profiler_start_error_code; // IDs 0-3 are reserved for bookkeeping slots, in this order. @@ -88,6 +95,7 @@ struct Names { profiler_func_stack_peak_buf(unique_name("profiler_func_stack_peak_buf")), profiler_func_kinds(unique_name("profiler_func_kinds")), profiler_func_buffer_func_ids(unique_name("profiler_func_buffer_func_ids")), + profiler_func_counters_approximated(unique_name("profiler_func_counters_approximated")), profiler_start_error_code(unique_name("profiler_start_error_code")) { // Reserve the bookkeeping slots first so their ids match the @@ -142,25 +150,6 @@ struct Names { display_name = f.profiler_display_name(); } } - } else if (kind == halide_profiler_func_kind_allocation && - starts_with(ir_name, "allocgroup__")) { - // Render "allocgroup__f1$0.0__f2$0.1.buffer" as - // "f1$0.0,f2$0.1.buffer": split off the "allocgroup" tag, - // join the rest with commas. - // - // TODO(next PR): once the per-Func counter machinery is - // back, attribute the bytes to each participating Func - // instead of presenting one combined row. The right place - // to thread that data through is a "declare_allocation" - // intrinsic emitted by FuseGPUThreadLoops at the position - // where it strips the Allocate, carrying the Func name, - // size, and MemoryType — the profiler can then bill the - // size to each Func via a hoistable counter (the GPU - // runtime has no memory_allocate hook, so we can't track - // it as a live allocation). - std::vector parts = split_string(ir_name, "__"); - parts.erase(parts.begin()); // drop the "allocgroup" tag - display_name = join_strings(parts, ","); } entry_info.push_back({display_name, ir_name, parent_id, canon, kind, buffer_func_id}); } @@ -187,6 +176,54 @@ struct Names { } }; +Expr compute_allocation_size(const vector &extents, + const Expr &condition, + const Type &type, + const std::string &name, + bool &can_fit_on_stack) { + can_fit_on_stack = true; + + Expr cond = simplify(condition); + if (is_const_zero(cond)) { + return make_zero(UInt(64)); + } + + int64_t constant_size = Allocate::constant_allocation_size(extents, name); + if (constant_size > 0) { + int64_t stack_bytes = constant_size * type.bytes(); + if (can_allocation_fit_on_stack(stack_bytes)) { + return make_const(UInt(64), stack_bytes); + } + } + + internal_assert(!extents.empty()); + + can_fit_on_stack = false; + Expr size = cast(extents[0]); + for (size_t i = 1; i < extents.size(); i++) { + size *= extents[i]; + } + size = simplify(Select::make(condition, size * type.bytes(), make_zero(UInt(64)))); + return size; +} + +// Unwrap a Broadcast(...) wrapper from an arg, then extract the Func name. +// declare_box_required_at_root / declare_stage carry the Func name as a +// StringImm in their first arg — the name is just a label for the +// profiler report, not a symbol that exists in scope, so using a +// StringImm keeps it from being matched by passes like +// InjectHostDevBufferCopies that substitute Variables named after Funcs. +// The arg can show up wrapped in a Broadcast after vectorization. +const std::string &handle_name(const Expr &e) { + const Expr *inner = &e; + if (const Broadcast *b = e.as()) { + inner = &b->value; + } + const StringImm *s = inner->as(); + internal_assert(s) << e; + return s->value; +} + // First pass: enumerate the entries (see file-level comment) and assign // each one an id. Walks every ProducerConsumer node (one entry per // producer, parented to the surrounding producer). @@ -267,6 +304,614 @@ class PreAllocateEntries : public IRMutator { } }; +// Second pass: inject counters for various stats as far outermost as possible, +// to minimize overhead. For example, instead of this: +// for (x in [min, max]) { +// increment_counter(..., 1); +// ... +// } +// We want to inject this: +// increment_counter(..., max - min + 1); +// for (x in [min, max]) { +// ... +// } +class InjectCounters : public IRMutator { +public: + InjectCounters(Names &names, const map &env) + : names(names), env(env) { + // The previous pass populated names.entry_info with every entry. + // Index them by IR name so declare_box_required_root (which + // carries the IR-level Func name from ScheduleFunctions) can + // find all entries for a Func. + for (int i = 0; i < names.num_ids(); i++) { + entries_by_name[names.entry_info[i].ir_name].push_back(i); + } + } + +protected: + Names &names; + const map &env; + + using IRMutator::visit; + + // ID of the currently-produced Func + int producer_id = -1; + + // Per-Func flag: are we currently inside that Func's pure def? + // Updated by the declare_stage marker that ScheduleFunctions emits + // at the start of each stage's loop nest (the marker also carries + // the Func name). A per-Func map handles compute_with cases where + // the stages of different Funcs are interleaved in the IR — each + // Store consults the flag for its own Func. + std::map func_in_pure_stage; + + // The counters we track. This list must be kept in sync with multiple other + // things. If you add a counter, also update: + // - the num_counters int below the enum + // - halide_profiler_update_counters in profiler_inlined.cpp + // - the fields of halide_profiler_func_stats in HalideRuntime.h + // - the block of code that prints counters to json in profiler_common.cpp + enum { MemoryTotal = 0, + NumAllocs, + ParallelLoops, + ParallelTasks, + PointsRequiredAtRoot, + PointsComputed }; + + static constexpr int num_counters = PointsComputed + 1; + + struct Counters { + + Expr counters[num_counters]; + + void add(const Counters &other) { + for (int i = 0; i < num_counters; i++) { + if (counters[i].defined()) { + if (other.counters[i].defined()) { + counters[i] += other.counters[i]; + } + } else { + counters[i] = other.counters[i]; + } + } + free_vars.insert(other.free_vars.begin(), other.free_vars.end()); + } + + void mul(const Expr &e) { + for (auto &counter : counters) { + if (counter.defined()) { + counter *= e; + } + } + add_free_vars(e); + } + + void count(int c, const Expr &e) { + internal_assert(e.defined() && e.type() == UInt(64)) << e; + if (counters[c].defined()) { + counters[c] += e; + } else { + counters[c] = e; + } + add_free_vars(e); + } + + void count(int c) { + count(c, make_one(UInt(64))); + } + + void add_free_vars(const Expr &e) { + visit_with(e, [&](auto *, const Variable *var) { + free_vars.insert(var->name); + }); + } + + // The free vars in the expressions + std::set free_vars; + }; + + const For *enclosing_loop = nullptr, *enclosing_parallel_loop = nullptr; + // True while mutating the body of any GPU loop. The CPU local_counters + // mechanism doesn't translate to GPU code (the buffer would have to be + // device-accessible, with atomic adds, and IHDBC has already run by the + // time we're injecting profiling). Instead, when in_gpu is set, we + // make any counter contribution that would normally have to flush + // mid-kernel hoist conservatively out of the kernel — substituting an + // upper bound for loop vars, wrapping in a Let for LetStmts, and + // wrapping in a Select for IfThenElse (or taking the max of the + // branches when the condition is impure). + bool in_gpu = false; + // thread-local counters + std::string local_counters; + // A map from a func id and a counter id to a slot in the local counters array + std::map, int> local_counters_indices; + + std::map counters; + + // entry id -> bitmask (over the num_counters counter slots) of counters + // that were summed as a conservative upper bound rather than exactly + // (see sum_counters_over_gpu_loop). Read by inject_profiling to fill the + // per-Func counters_approximated field. + std::map counters_approximated; + + // name -> all entry ids with that name. Built once in the constructor; + // only declare_box_required_root reads it. + std::map> entries_by_name; + + bool is_func(const std::string &name) const { + return env.find(name) != env.end(); + } + + Stmt flush(const Stmt &s, int id, const Counters &c) { + if (enclosing_loop && + enclosing_parallel_loop && + enclosing_loop != enclosing_parallel_loop) { + // Flush to local counters + if (local_counters.empty()) { + local_counters = unique_name("local_counters"); + } + std::vector stores; + stores.reserve(num_counters); + for (int i = 0; i < num_counters; i++) { + if (!c.counters[i].defined()) { + continue; + } + int n = (int)local_counters_indices.size(); + int idx = + local_counters_indices.try_emplace({id, i}, n).first->second; + Expr old = Load::make(UInt(64), local_counters, idx); + stores.push_back(Store::make(local_counters, old + c.counters[i], idx)); + } + stores.push_back(s); + return Block::make(stores); + } else { + // Flush to global counters + std::vector args(2 + num_counters); + args[0] = Variable::make(Handle(), names.profiler_instance); + args[1] = id; + for (int i = 0; i < num_counters; i++) { + Expr count = c.counters[i]; + args[i + 2] = count.defined() ? count : make_zero(UInt(64)); + } + Expr call = Call::make(Int(32), "halide_profiler_update_counters", args, Call::Extern); + return Block::make(Evaluate::make(std::move(call)), s); + } + } + + Stmt flush_all(const Stmt &stmt) { + Stmt s = stmt; + for (const auto &p : counters) { + s = flush(s, p.first, p.second); + } + counters.clear(); + return s; + } + + Stmt flush_all_that_depend_on_var(const Stmt &stmt, const std::string &var) { + Stmt s = stmt; + for (auto it = counters.begin(); it != counters.end();) { + const auto &[id, c] = *it; + if (c.free_vars.count(var)) { + s = flush(s, id, c); + it = counters.erase(it); + } else { + it++; + } + } + return s; + } + + void merge(const std::map &other) { + for (const auto &it : other) { + counters[it.first].add(it.second); + } + } + + // Recompute a Counters object's free_vars set from its current Exprs. + // Used after any hoisting operation that mutates the Exprs in place. + static void recompute_free_vars(Counters &c) { + c.free_vars.clear(); + for (const auto &counter : c.counters) { + if (counter.defined()) { + c.add_free_vars(counter); + } + } + } + + // GPU can't flush counters mid-kernel, so sum each counter over a + // closing-out loop symbolically (in place of the mul-by-extent used for + // CPU loops). We bound the per-iteration value by its max, and the + // number of contributing iterations by the loop-clipped range where the + // value can be non-zero (0 < counter, since counters are non-negative). + // solve_for_outer_interval over-approximates that range, so the result + // stays a conservative upper bound — far tighter than max-value × + // full-extent for a footprint-guarded contribution like + // select(guard, k, 0), which the simplifier reduces to the bare guard. + // A counter that doesn't depend on the loop var falls out as its value × + // the full extent. If no finite value bound exists we drop it (an + // under-estimate we accept over a bogus huge number). + void sum_counters_over_gpu_loop(const For *op, const Expr &extent) { + const std::string &var = op->name; + Interval loop_bounds(op->min, simplify(op->min + extent - 1)); + Scope scope; + scope.push(var, loop_bounds); + for (auto &[id, c] : counters) { + for (int ci = 0; ci < num_counters; ci++) { + Expr &counter = c.counters[ci]; + if (!counter.defined()) { + continue; + } + // A counter that varies over the loop is summed as val.max × + // (contributing width), a conservative upper bound rather than + // an exact sum, so flag it. (An invariant counter reduces to + // its exact value × extent below.) + if (expr_uses_var(counter, var)) { + counters_approximated[id] |= (1u << ci); + } + Interval val = bounds_of_expr_in_scope(counter, scope); + if (!val.has_upper_bound()) { + counter = Expr(); + continue; + } + Interval support = solve_for_outer_interval( + simplify(make_zero(counter.type()) < counter), var); + support = Interval::make_intersection(support, loop_bounds); + Expr width = clamp(simplify(support.max - support.min + 1), 0, extent); + counter = simplify(val.max * cast(UInt(64), width)); + } + recompute_free_vars(c); + } + } + + // GPU hoisting: when a LetStmt is closing out, any counter contribution + // that depends on the let-bound name gets wrapped in an exact Let — + // but only if the RHS is pure. An impure RHS (e.g. a Load whose + // backing buffer may be mutated, or a non-pure Call) would be + // re-evaluated in a different scope by the wrapped Let, which can + // change its meaning. In that case we drop the contribution and mark + // the entry approximated. + void hoist_let(const std::string &name, const Expr &value) { + bool value_pure = is_pure(value); + for (auto &[id, c] : counters) { + if (!c.free_vars.count(name)) { + continue; + } + for (auto &counter : c.counters) { + if (counter.defined() && expr_uses_var(counter, name)) { + if (value_pure) { + counter = Let::make(name, value, counter); + } else { + counter = Expr(); + } + } + } + recompute_free_vars(c); + } + } + + // GPU hoisting: combine the then- and else-branch counter contributions + // of an IfThenElse into the outer scope. For a pure condition, exact via + // Select. For an impure condition (e.g. a Load), upper-bound the + // contribution by max(then, else) (the branches are mutually exclusive) + // and mark the entries as approximated. + void hoist_if(const Expr &condition, + std::map &then_counters, + std::map &else_counters) { + bool cond_pure = is_pure(condition); + std::set ids; + for (const auto &p : then_counters) { + ids.insert(p.first); + } + for (const auto &p : else_counters) { + ids.insert(p.first); + } + for (int id : ids) { + Counters merged; + auto *t = then_counters.count(id) ? &then_counters[id] : nullptr; + auto *e = else_counters.count(id) ? &else_counters[id] : nullptr; + for (int i = 0; i < num_counters; i++) { + Expr tv = (t && t->counters[i].defined()) ? t->counters[i] : Expr(); + Expr ev = (e && e->counters[i].defined()) ? e->counters[i] : Expr(); + if (!tv.defined() && !ev.defined()) { + continue; + } + Expr zero64 = make_zero(UInt(64)); + Expr ti = tv.defined() ? tv : zero64; + Expr ei = ev.defined() ? ev : zero64; + if (cond_pure) { + merged.counters[i] = select(condition, ti, ei); + } else { + // Branches are mutually exclusive — only one runs per + // execution — so the tight conservative upper bound on + // the contribution is max(then, else). + merged.counters[i] = max(ti, ei); + } + } + recompute_free_vars(merged); + counters[id].add(merged); + } + } + + // Compute the total number of points in a box passed to declare_box_required*. + // The args after the func handle are (min, max) pairs per dim; the result is + // a scalar UInt(64) total, reduced across any surrounding vector lanes. + static Expr box_total(const Call *op) { + int lanes = op->type.lanes(); + Expr total = make_one(UInt(64, lanes)); + for (size_t i = 1; i < op->args.size(); i += 2) { + total *= cast(total.type(), op->args[i + 1] + 1 - op->args[i]); + } + if (lanes > 1) { + total = VectorReduce::make(VectorReduce::Add, total, 1); + } + // Simplifying here removes false dependences on loop vars. + return simplify(total); + } + + Expr visit(const Call *op) override { + if (op->is_intrinsic(Call::declare_box_required_at_root)) { + // Bill the pipeline-wide root box to this Func's canonical + // entry only. It's a Func-level fact, not a per-entry one, so + // summing it across entries would over-count. The reporter + // looks it up via fs->canonical_id when computing each entry's + // local recompute ratio. + auto it = entries_by_name.find(handle_name(op->args[0])); + if (it != entries_by_name.end()) { + // entries_by_name was filled in id-ascending order, and the + // canonical id is the first id allocated for the name, so + // it->second.front() is always the canonical id. + counters[it->second.front()].count(PointsRequiredAtRoot, box_total(op)); + } + return make_zero(op->type); + } else if (op->is_intrinsic(Call::declare_allocation)) { + internal_assert(op->args.size() == 3); + std::string fname = names.prefix(handle_name(op->args[0])); + auto eit = env.find(fname); + if (eit != env.end() && !eit->second.should_not_profile()) { + int id = names.id_for_entry(fname, producer_id); + counters[id].count(NumAllocs); + counters[id].count(MemoryTotal, cast(UInt(64), op->args[1])); + } + // Leave the marker in the IR (rather than stripping it) so + // InjectProfiling can also emit the memory_current/peak + // tracking calls for it — those aren't counters and can't be + // handled here. + return op; + } else if (op->is_intrinsic(Call::declare_stage)) { + // Marker from ScheduleFunctions saying "we're starting stage N + // of Func F here". Update our per-Func pure-def flag and strip + // the marker from the IR. + internal_assert(op->args.size() == 2); + auto stage = as_const_int(op->args[1]); + internal_assert(stage); + func_in_pure_stage[handle_name(op->args[0])] = (*stage == 0); + return make_zero(op->type); + } else { + // Counter events are never nested, so no recursive mutate call. + return op; + } + } + + // True for Stores to buffers we want to bill: a Func's own storage, or + // an output parameter. False for internal bookkeeping buffers like + // storage-folding head trackers, async semaphores, and sampling + // tokens — we don't want their stores inflating points_computed. + bool is_real_data_buffer(const Store *op) const { + return op->param.defined() || is_func(names.prefix(op->name)); + } + + Stmt visit(const Store *op) override { + if (is_real_data_buffer(op)) { + std::string f = names.prefix(op->name); + // Stores in a producer block are to the Func being produced, so + // bill them to the current producer's entry id. (That's the + // right entry even if f has multiple entries elsewhere.) + int id = (producer_id >= 0 && names.entry_info[producer_id].ir_name == f) ? + producer_id : + names.id_for_name(f); + Counters &c = counters[id]; + int lanes = op->value.type().lanes(); + // Only the pure def (stage 0) contributes to "points computed"; + // update-def stores are a separate kind of work and shouldn't + // show up as recompute. For Tuple-valued Funcs each output + // point produces one Store per tuple element (to buffers + // f.0, f.1, ...), so counting all of them would inflate + // points_computed by the tuple arity. Skip any store whose + // buffer name's final dotted component parses as a non-zero + // integer -- those are the non-canonical tuple elements. + auto it = func_in_pure_stage.find(f); + if (it != func_in_pure_stage.end() && it->second) { + size_t last_dot = op->name.rfind('.'); + if (last_dot == std::string::npos || ends_with(op->name, ".0")) { + c.count(PointsComputed, make_const(UInt(64), lanes)); + } + } + } + return IRMutator::visit(op); + } + + Stmt visit(const ProducerConsumer *op) override { + if (op->is_producer) { + // One entry per producer node, parented to the surrounding + // producer. See file-level comment for why this matters. + int id = names.id_for_entry(op->name, producer_id); + ScopedValue old(producer_id, id); + return IRMutator::visit(op); + } else { + return IRMutator::visit(op); + } + } + + Stmt visit(const Allocate *op) override { + // Bill heap allocations to NumAllocs and MemoryTotal. + std::string fname = names.prefix(op->name); + auto eit = env.find(fname); + if (eit != env.end() && !eit->second.should_not_profile()) { + bool can_fit_on_stack; + Expr size = compute_allocation_size(op->extents, op->condition, + op->type, op->name, can_fit_on_stack); + bool on_stack = can_fit_on_stack && !op->new_expr.defined(); + if (!is_const_zero(size) && !on_stack) { + int id = names.id_for_entry(fname, producer_id); + counters[id].count(NumAllocs, cast(UInt(64), op->condition)); + counters[id].count(MemoryTotal, size); + } + } + return IRMutator::visit(op); + } + + Stmt visit(const For *op) override { + // GPU loops are also is_unordered_parallel(). We can't use the CPU + // local_counters mechanism inside a GPU kernel (device memory, + // atomics, IHDBC ordering), so when in_gpu we hoist any + // closing-out contributions instead of flushing — see the helper + // methods above. + ScopedValue bind_gpu(in_gpu, in_gpu || is_gpu(op->for_type)); + + decltype(counters) old; + old.swap(counters); + + Stmt body; + { + ScopedValue bind1(enclosing_loop, op); + ScopedValue bind2(enclosing_parallel_loop, + op->is_unordered_parallel() ? + op : + enclosing_parallel_loop); + body = mutate(op->body); + } + + if (op->is_unordered_parallel() && + !local_counters.empty()) { + // Flush any thread-local counters to global state + std::map to_flush; + for (auto [p, idx] : local_counters_indices) { + auto [id, counter] = p; + to_flush[id].counters[counter] = + Load::make(UInt(64), local_counters, idx); + } + + counters.swap(to_flush); + Stmt post_flush = flush_all(Evaluate::make(0)); + counters.swap(to_flush); + + std::vector stmts; + stmts.reserve(local_counters_indices.size() + 2); + for (int i = 0; i < (int)local_counters_indices.size(); i++) { + stmts.push_back(Store::make(local_counters, make_zero(UInt(64)), i)); + } + + stmts.push_back(std::move(body)); + stmts.push_back(post_flush); + body = Block::make(stmts); + + Expr size = (int)local_counters_indices.size(); + body = Allocate::make(local_counters, UInt(64), MemoryType::Stack, {size}, const_true(), body); + } + + // Scale up the counters by the loop trip count + Expr e = simplify(op->extent()); + + if (in_gpu) { + // Can't flush in the middle of a GPU kernel — sum each counter + // over the loop symbolically (this subsumes the mul-by-extent + // done for CPU loops below). + sum_counters_over_gpu_loop(op, e); + } else { + body = flush_all_that_depend_on_var(body, op->name); + for (auto &[_, c] : counters) { + c.mul(e); + } + } + + merge(old); + + if (op->is_unordered_parallel()) { + // The parallel loop belongs to the currently-producing Func. + int id = producer_id >= 0 ? producer_id : names.id_for_name(names.prefix(op->name)); + + if (!bind_gpu.old_value) { + // Unless this is an inner GPU loop, this counts as a parallel loop launch + counters[id].count(ParallelLoops); + } + counters[id].count(ParallelTasks, cast(UInt(64), e)); + } + + return For::make(op->name, op->min, op->max, + op->for_type, op->partition_policy, op->device_api, std::move(body)); + } + + Stmt visit(const LetStmt *op) override { + decltype(counters) old; + counters.swap(old); + Stmt body = mutate(op->body); + if (in_gpu) { + hoist_let(op->name, op->value); + } else { + body = flush_all_that_depend_on_var(body, op->name); + } + merge(old); + return LetStmt::make(op->name, op->value, body); + } + + Stmt visit(const IfThenElse *op) override { + if (in_gpu) { + // Inside a GPU kernel we can't flush in the branches; instead + // combine the branch contributions into the outer scope via + // Select (or a conservative max of the branches if the + // condition is impure). + decltype(counters) outer; + counters.swap(outer); + Stmt then_case = mutate(op->then_case); + decltype(counters) then_counters; + counters.swap(then_counters); + Stmt else_case; + decltype(counters) else_counters; + if (op->else_case.defined()) { + else_case = mutate(op->else_case); + counters.swap(else_counters); + } + counters.swap(outer); + hoist_if(op->condition, then_counters, else_counters); + return IfThenElse::make(op->condition, then_case, else_case); + } + + decltype(counters) old; + counters.swap(old); + Stmt then_case, else_case; + then_case = mutate(op->then_case); + then_case = flush_all(then_case); + if (op->else_case.defined()) { + else_case = mutate(op->else_case); + else_case = flush_all(else_case); + } + counters.swap(old); + return IfThenElse::make(op->condition, then_case, else_case); + } + + Stmt visit(const Block *op) override { + // Put the outermost counter update just outside the timing start + const Evaluate *eval = op->first.as(); + const Call *call = eval ? eval->value.as() : nullptr; + if (call && call->is_intrinsic(Call::profiling_enable_instance_marker)) { + return flush_all(IRMutator::visit(op)); + } else { + return IRMutator::visit(op); + } + } + +public: + Stmt operator()(const Stmt &s) { + return flush_all(IRMutator::operator()(s)); + } + + // Counter-approximation bitmask for entry `id` (0 if all exact). + uint32_t approximated_counters(int id) const { + auto it = counters_approximated.find(id); + return it == counters_approximated.end() ? 0 : it->second; + } +}; + class InjectProfiling : public IRMutator { // Thread-activity tracking around parallel constructs and sampling-token // plumbing for leaf parallel tasks. @@ -414,6 +1059,16 @@ class InjectProfiling : public IRMutator { return s; } + // Bill a heap allocation of `size` bytes to entry `idx`, bumping its + // memory_current/peak. Shared by the Allocate visitor and the + // declare_allocation marker (device-only buffers whose host Allocate + // was nulled out). num_allocs/memory_total are handled separately via + // the counter path. + Expr memory_allocate_call(int idx, const Expr &size) { + return Call::make(Int(32), "halide_profiler_memory_allocate", + {profiler_instance, idx, size}, Call::Extern); + } + Stmt set_current_func(int id) { if (most_recently_set_func == id) { return Evaluate::make(0); @@ -427,44 +1082,33 @@ class InjectProfiling : public IRMutator { return s; } - Expr compute_allocation_size(const vector &extents, - const Expr &condition, - const Type &type, - const std::string &name, - bool &can_fit_on_stack) { - can_fit_on_stack = true; - - Expr cond = simplify(condition); - if (is_const_zero(cond)) { // Condition always false - return make_zero(UInt(64)); - } - - int64_t constant_size = Allocate::constant_allocation_size(extents, name); - if (constant_size > 0) { - int64_t stack_bytes = constant_size * type.bytes(); - if (can_allocation_fit_on_stack(stack_bytes)) { // Allocation on stack - return make_const(UInt(64), stack_bytes); - } - } - - // Check that the allocation is not scalar (if it were scalar - // it would have constant size). - internal_assert(!extents.empty()); - - can_fit_on_stack = false; - Expr size = cast(extents[0]); - for (size_t i = 1; i < extents.size(); i++) { - size *= extents[i]; - } - size = simplify(Select::make(condition, size * type.bytes(), make_zero(UInt(64)))); - return size; - } - Expr visit(const Call *op) override { if (op->is_intrinsic(Call::profiling_enable_instance_marker)) { // End of the bounds-query prelude — start collecting samples. return Call::make(Int(32), "halide_profiler_enable_instance", {profiler_instance}, Call::Extern); + } else if (op->is_intrinsic(Call::declare_allocation)) { + // A device-only buffer: InjectHostDevBufferCopies nulled its + // host Allocate (condition false), so visit(Allocate) pushed a + // zero-size func_alloc_sizes entry and emitted no tracking. The + // device storage is real, and its Free node still brackets the + // lifetime, so rewrite the entry to the device size — the + // matching Free then emits a memory_free — and emit the + // memory_allocate here. (num_allocs/memory_total are billed via + // the counter path.) + internal_assert(op->args.size() == 3); + std::string name = handle_name(op->args[0]); + Expr size = simplify(cast(op->args[1])); + int idx = -1; + if (func_alloc_sizes.contains(name)) { + idx = func_alloc_sizes.get(name).id; + func_alloc_sizes.pop(name); + } + func_alloc_sizes.push(name, {/*on_stack=*/false, size, idx}); + if (profiling_memory && idx >= 0 && !is_const_zero(size)) { + return memory_allocate_call(idx, size); + } + return make_zero(op->type); } else { return IRMutator::visit(op); } @@ -494,12 +1138,8 @@ class InjectProfiling : public IRMutator { idx = stack.back(); break; case Kind::NotAFunc: - // Allocations whose name doesn't match a Func (e.g. fused - // allocation-group buffers) still get tracked: mint an - // allocation-kind entry under the current producer so the - // bytes are attributed somewhere. - idx = names.id_for_entry(op->name, stack.back() == names.overhead_id ? -1 : stack.back(), - halide_profiler_func_kind_allocation); + // Ignore allocations that don't correspond to a Func + idx = -1; break; } @@ -520,15 +1160,14 @@ class InjectProfiling : public IRMutator { } vector tasks; - bool track_heap_allocation = !is_const_zero(size) && !on_stack && profiling_memory; + bool track_heap_allocation = !is_const_zero(size) && !on_stack && profiling_memory && idx >= 0; if (track_heap_allocation) { debug(3) << " Allocation on heap: " << op->name << "(" << size << ") in pipeline " << names.pipeline_name << "\n"; tasks.push_back(set_current_func(names.malloc_id)); - tasks.push_back(Evaluate::make(Call::make(Int(32), "halide_profiler_memory_allocate", - {profiler_instance, idx, size}, Call::Extern))); + tasks.push_back(Evaluate::make(memory_allocate_call(idx, size))); } Stmt body = mutate(op->body); @@ -564,7 +1203,7 @@ class InjectProfiling : public IRMutator { if (!is_const_zero(alloc.size)) { int idx = alloc.id; if (!alloc.on_stack) { - if (profiling_memory) { + if (profiling_memory && idx >= 0) { debug(3) << " Free on heap: " << op->name << "(" << alloc.size << ") in pipeline " << names.pipeline_name << "\n"; vector tasks{ @@ -818,15 +1457,20 @@ class InjectProfiling : public IRMutator { } // namespace -Stmt inject_profiling(const Stmt &stmt, const string &pipeline_name, const std::map &env) { +Stmt inject_profiling(const Stmt &stmt, const string &pipeline_name, const std::map &env, const Target &target) { Names names(pipeline_name, env); // 1) Allocate an id for every entry. After this, names.entry_info // is the full set of entries we'll report on. Stmt s = PreAllocateEntries(names, env)(stmt); - // 2) Inject the profiler scaffolding: thread activation, memory - // tracking, current-func tracking, copy-to-host/device timing. + // 2) Inject the counter-update calls for stats (parallel loops, + // points computed, etc.). + InjectCounters injector(names, env); + s = injector(s); + + // 3) Inject the rest of the profiler scaffolding: thread activation, + // memory tracking, current-func tracking, copy-to-host/device timing. InjectProfiling profiling(names, env); s = profiling(s); @@ -839,6 +1483,7 @@ Stmt inject_profiling(const Stmt &stmt, const string &pipeline_name, const std:: Expr func_canonical_ids_buf = Variable::make(Handle(), names.profiler_func_canonical_ids); Expr func_kinds_buf = Variable::make(Handle(), names.profiler_func_kinds); Expr func_buffer_func_ids_buf = Variable::make(Handle(), names.profiler_func_buffer_func_ids); + Expr func_counters_approximated_buf = Variable::make(Handle(), names.profiler_func_counters_approximated); Expr start_profiler = Call::make(Int(32), "halide_profiler_instance_start", {pipeline_name, @@ -848,6 +1493,8 @@ Stmt inject_profiling(const Stmt &stmt, const string &pipeline_name, const std:: func_canonical_ids_buf, func_kinds_buf, func_buffer_func_ids_buf, + func_counters_approximated_buf, + make_const(UInt(64), target.natural_vector_size(UInt(8))), instance}, Call::Extern); @@ -898,6 +1545,7 @@ Stmt inject_profiling(const Stmt &stmt, const string &pipeline_name, const std:: std::vector func_canonical_ids(num_funcs); std::vector func_kinds(num_funcs); std::vector func_buffer_func_ids(num_funcs); + std::vector func_counters_approximated(num_funcs); for (int i = 0; i < num_funcs; i++) { const auto &info = names.entry_info[i]; func_names[i] = info.name; @@ -905,6 +1553,7 @@ Stmt inject_profiling(const Stmt &stmt, const string &pipeline_name, const std:: func_canonical_ids[i] = info.canonical_id; func_kinds[i] = make_const(Int(32), (int)info.kind); func_buffer_func_ids[i] = info.buffer_func_id; + func_counters_approximated[i] = make_const(UInt(32), injector.approximated_counters(i)); } s = LetStmt::make(names.profiler_func_names, Call::make(Handle(), Call::make_struct, func_names, Call::Intrinsic), s); @@ -912,6 +1561,7 @@ Stmt inject_profiling(const Stmt &stmt, const string &pipeline_name, const std:: s = LetStmt::make(names.profiler_func_canonical_ids, Call::make(Handle(), Call::make_struct, func_canonical_ids, Call::Intrinsic), s); s = LetStmt::make(names.profiler_func_kinds, Call::make(Handle(), Call::make_struct, func_kinds, Call::Intrinsic), s); s = LetStmt::make(names.profiler_func_buffer_func_ids, Call::make(Handle(), Call::make_struct, func_buffer_func_ids, Call::Intrinsic), s); + s = LetStmt::make(names.profiler_func_counters_approximated, Call::make(Handle(), Call::make_struct, func_counters_approximated, Call::Intrinsic), s); s = Block::make(Evaluate::make(stop_profiler), s); // Allocate memory for the profiler instance state diff --git a/src/Profiling.h b/src/Profiling.h index b91644409655..9e05a7e8f2a2 100644 --- a/src/Profiling.h +++ b/src/Profiling.h @@ -29,6 +29,9 @@ #include "Expr.h" namespace Halide { + +struct Target; + namespace Internal { class Function; @@ -40,7 +43,7 @@ class Function; * storage flattening, but after all bounds inference. * */ -Stmt inject_profiling(const Stmt &, const std::string &, const std::map &env); +Stmt inject_profiling(const Stmt &, const std::string &, const std::map &env, const Target &target); } // namespace Internal } // namespace Halide diff --git a/src/ScheduleFunctions.cpp b/src/ScheduleFunctions.cpp index 53d94f3382a9..d024d04ed12a 100644 --- a/src/ScheduleFunctions.cpp +++ b/src/ScheduleFunctions.cpp @@ -1223,7 +1223,13 @@ class InjectFunctionRealization : public IRMutator { } Stmt operator()(const Stmt &stmt) { - return IRMutator::operator()(stmt); + Stmt s = IRMutator::operator()(stmt); + if (target.has_feature(Target::Profile)) { + for (const auto &func : funcs) { + s = declare_box(s, func, Call::declare_box_required_at_root); + } + } + return s; } protected: @@ -1241,7 +1247,12 @@ class InjectFunctionRealization : public IRMutator { // substitute names of in-scope buffers (most notably box_touched // analysis itself) follow that name. Stmt declare_box(const Stmt &stmt, const Function &f, Call::IntrinsicOp intrin) { - Expr name_arg = Variable::make(Handle(), f.name()); + Expr name_arg; + if (intrin == Call::declare_box_touched) { + name_arg = Variable::make(Handle(), f.name()); + } else { + name_arg = Expr(f.name()); + } std::vector args; args.reserve(2 * f.dimensions() + 1); args.push_back(std::move(name_arg)); @@ -1844,6 +1855,18 @@ class InjectFunctionRealization : public IRMutator { Stmt produce_def = build_produce_definition(f, def_prefix, def, func_stage.second > 0, replacements, add_lets, aliases); + if (target.has_feature(Target::Profile)) { + // Mark the start of this Func's stage so InjectCounters can + // distinguish pure-def stores from update-def stores even + // when there's no surrounding For loop with a stage-named + // var to key off (zero-dimensional or fully-unrolled Funcs, + // or stages of Funcs whose pure def has no Vars). + Expr marker = Call::make(Int(32), Call::declare_stage, + {Expr(f.name()), + make_const(Int(32), func_stage.second)}, + Call::Intrinsic); + produce_def = Block::make(Evaluate::make(marker), produce_def); + } producer = inject_stmt(producer, produce_def, def.schedule().fuse_level().level); } diff --git a/src/Simplify_LT.cpp b/src/Simplify_LT.cpp index fc6145004669..cac0a804445f 100644 --- a/src/Simplify_LT.cpp +++ b/src/Simplify_LT.cpp @@ -49,6 +49,9 @@ Expr Simplify::visit(const LT *op, ExprInfo *info) { rewrite(broadcast(x, c0) < broadcast(y, c0), broadcast(x < y, c0)) || + rewrite(c0 < select(x, c1, c2), select(x, fold(c0 < c1), fold(c0 < c2))) || + rewrite(select(x, c1, c2) < c0, select(x, fold(c1 < c0), fold(c2 < c0))) || + // We can learn more from equality than less with (Euclidean) mod. (!ty.is_float() && EVAL_IN_LAMBDA // (rewrite(x % y < 1, x % y == 0) || @@ -266,9 +269,6 @@ Expr Simplify::visit(const LT *op, ExprInfo *info) { rewrite(select(y, z, x + c0) < x + c1, y && (z < x + c1), c0 >= c1) || rewrite(select(y, z, x + c0) < x + c1, !y || (z < x + c1), c0 < c1) || - rewrite(c0 < select(x, c1, c2), select(x, fold(c0 < c1), fold(c0 < c2))) || - rewrite(select(x, c1, c2) < c0, select(x, fold(c1 < c0), fold(c2 < c0))) || - // Normalize comparison of ramps to a comparison of a ramp and a broadacst rewrite(ramp(x, y, lanes) < ramp(z, w, lanes), ramp(x - z, y - w, lanes) < 0) || diff --git a/src/StorageFolding.cpp b/src/StorageFolding.cpp index c076944c70c4..3ab1e46727ac 100644 --- a/src/StorageFolding.cpp +++ b/src/StorageFolding.cpp @@ -678,11 +678,11 @@ class AttemptStorageFoldingOfFunction : public IRMutator { // consumer wants to move the counter, it must // also acquire or release the semaphore to // prevent them from diverging too far. - dynamic_footprint = func.name() + ".folding_semaphore." + op->name + unique_name('_'); + dynamic_footprint = unique_name("fold") + "." + func.name() + "." + op->name + ".semaphore"; head = dynamic_footprint + ".head"; tail = dynamic_footprint + ".tail"; } else { - dynamic_footprint = func.name() + "." + op->name + unique_name('_') + ".head"; + dynamic_footprint = unique_name("fold") + "." + func.name() + "." + op->name + ".head"; head = tail = dynamic_footprint; } diff --git a/src/VectorizeLoops.cpp b/src/VectorizeLoops.cpp index 20e84b65a17e..84429f076cf3 100644 --- a/src/VectorizeLoops.cpp +++ b/src/VectorizeLoops.cpp @@ -547,6 +547,18 @@ class VectorSubs : public IRMutator { max_lanes = std::max(new_arg.type().lanes(), max_lanes); } + // Profiler counter markers (declare_box_required_at_root) carry per-lane + // counter contributions encoded in their type's lane count, so they + // must be widened to the full lane count of the surrounding + // vectorized loop even when their args don't reference any + // vectorized vars. + if (op->is_intrinsic({Call::declare_box_required_at_root})) { + max_lanes = 1; + for (const auto &vv : vectorized_vars) { + max_lanes *= vv.lanes; + } + } + if (!changed && max_lanes <= 1) { return op; } else if (op->name == Call::trace) { diff --git a/src/runtime/HalideRuntime.h b/src/runtime/HalideRuntime.h index 4ea34d4dc1b6..2a717063b900 100644 --- a/src/runtime/HalideRuntime.h +++ b/src/runtime/HalideRuntime.h @@ -2024,6 +2024,19 @@ struct HALIDE_ATTRIBUTE_ALIGN(8) halide_profiler_func_stats { * canonical id of the Func whose buffer is being copied. -1 otherwise. */ int buffer_func_id; + /** A bitmask flagging which of this Func's aggregated counters are + * conservative upper bounds rather than exact values. The bits index the + * counters passed to halide_profiler_update_counters, in that order: + * bit 0 = memory_total, 1 = num_allocs, 2 = parallel_loops, + * 3 = parallel_tasks, 4 = points_required_at_root, 5 = points_computed. + * (active_threads_numerator/denominator are sampled at runtime rather + * than summed over loops, so they are never approximated and have no + * bit.) A set bit only happens on GPU, where a guarded contribution + * can't be summed exactly and is bounded instead; the reporter marks + * such columns with a leading '<'. Must stay in sync with the counter + * enum in src/Profiling.cpp. */ + uint32_t counters_approximated; + /** Total time taken evaluating this Func (in nanoseconds). */ uint64_t HALIDE_ATTRIBUTE_ALIGN(8) time; @@ -2036,6 +2049,9 @@ struct HALIDE_ATTRIBUTE_ALIGN(8) halide_profiler_func_stats { /** The peak stack allocation of this Func's threads. */ uint64_t stack_peak; + // Everything field after this point is a counter. They are aggregated by + // blindly adding. + /** The total memory allocation of this Func. */ uint64_t memory_total; @@ -2045,6 +2061,27 @@ struct HALIDE_ATTRIBUTE_ALIGN(8) halide_profiler_func_stats { /** The total number of times heap storage for this Func was allocated. */ uint64_t num_allocs; + + /** The number of parallel loops launched to compute some of this + * Func. I.e. the number of times halide_do_par_for was called due to one of + * this Func's parallel loops. Next, the total number of iterations of those + * loops. */ + uint64_t parallel_loops, parallel_tasks; + + /** The number of points required of this Func at root. Will be less than + * points_required when there is redundant recompute due to use of + * compute_at. */ + uint64_t points_required_at_root; + + /** The number of points actually computed by this Func's pure + * definition (its stage-0 stores), weighted by vector lane count. + * Captures forms of over-computation that the box-required counters + * miss: tail strategies like RoundUp that write past the requested + * extent, and cases where sliding-window failed so each produce-node + * iteration computes the full required box. Counting just stage-0 + * stores keeps update definitions from being conflated as + * "recompute". */ + uint64_t points_computed; }; /** Per-pipeline state tracked by the sampling profiler. These exist @@ -2066,6 +2103,10 @@ struct HALIDE_ATTRIBUTE_ALIGN(8) halide_profiler_pipeline_stats { * work while computing this pipeline. */ uint64_t active_threads_numerator, active_threads_denominator; + /** The native vector width for the target this pipeline ran on, in + * bytes. This is used to drive some performance warnings. */ + uint64_t native_vector_bytes; + /** The name of this pipeline. A global constant string. */ const char *name; diff --git a/src/runtime/profiler_common.cpp b/src/runtime/profiler_common.cpp index 42511f41636d..9bb35e5bb6f7 100644 --- a/src/runtime/profiler_common.cpp +++ b/src/runtime/profiler_common.cpp @@ -68,7 +68,8 @@ WEAK halide_profiler_pipeline_stats *find_or_create_pipeline(const char *pipelin const int *func_parents, const int *func_canonical_ids, const int *func_kinds, - const int *func_buffer_func_ids) { + const int *func_buffer_func_ids, + const uint32_t *func_counters_approximated) { halide_profiler_state *s = halide_profiler_get_state(); for (halide_profiler_pipeline_stats *p = s->pipelines; p; @@ -103,6 +104,7 @@ WEAK halide_profiler_pipeline_stats *find_or_create_pipeline(const char *pipelin p->funcs[i].canonical_id = func_canonical_ids[i]; p->funcs[i].kind = (halide_profiler_func_kind)func_kinds[i]; p->funcs[i].buffer_func_id = func_buffer_func_ids[i]; + p->funcs[i].counters_approximated = func_counters_approximated[i]; } s->pipelines = p; return p; @@ -249,6 +251,8 @@ WEAK int halide_profiler_instance_start(void *user_context, const int *func_canonical_ids, const int *func_kinds, const int *func_buffer_func_ids, + const uint32_t *func_counters_approximated, + uint64_t native_vector_bytes, halide_profiler_instance_state *instance) { // Tell the instance where we stashed the per-func state - just after the // instance itself. @@ -288,11 +292,13 @@ WEAK int halide_profiler_instance_start(void *user_context, halide_profiler_pipeline_stats *p = find_or_create_pipeline(pipeline_name, num_funcs, func_names, func_parents, func_canonical_ids, - func_kinds, func_buffer_func_ids); + func_kinds, func_buffer_func_ids, + func_counters_approximated); if (!p) { // Allocating space to track the statistics failed. return halide_error_out_of_memory(user_context); } + p->native_vector_bytes = native_vector_bytes; // Tell the instance the pipeline to which it belongs. instance->pipeline_stats = p; @@ -436,15 +442,10 @@ WEAK void halide_profiler_memory_allocate(void *user_context, // does not free the structs unless user specifically calls // halide_profiler_reset(). - // Update per-instance memory stats - atomic_add_fetch_sequentially_consistent(&instance->num_allocs, 1); - atomic_add_fetch_sequentially_consistent(&instance->memory_total, incr); + // num_allocs and memory_total go through halide_profiler_update_counters. uint64_t p_mem_current = atomic_add_fetch_sequentially_consistent(&instance->memory_current, incr); sync_compare_max_and_swap(&instance->memory_peak, p_mem_current); - // Update per-func memory stats - atomic_add_fetch_sequentially_consistent(&func->num_allocs, 1); - atomic_add_fetch_sequentially_consistent(&func->memory_total, incr); uint64_t f_mem_current = atomic_add_fetch_sequentially_consistent(&func->memory_current, incr); sync_compare_max_and_swap(&func->memory_peak, f_mem_current); } @@ -480,6 +481,21 @@ WEAK void halide_profiler_memory_free(void *user_context, atomic_sub_fetch_sequentially_consistent(&func->memory_current, decr); } +// Bit positions in halide_profiler_func_stats::counters_approximated. Must +// stay in sync with the counter enum in src/Profiling.cpp. +enum { + counter_memory_total = 0, + counter_num_allocs = 1, + counter_parallel_loops = 2, + counter_parallel_tasks = 3, + counter_points_required_at_root = 4, + counter_points_computed = 5, +}; + +ALWAYS_INLINE bool counter_is_approximate(const halide_profiler_func_stats *fs, int counter) { + return (fs->counters_approximated & (1u << counter)) != 0; +} + WEAK void halide_profiler_report_unlocked(void *user_context, halide_profiler_state *s) { StringStreamPrinter<1024> sstr(user_context); @@ -592,7 +608,9 @@ WEAK void halide_profiler_report_unlocked(void *user_context, halide_profiler_st }; // SI-suffixed counter (10000 -> 10K, 1e6 -> 1.0M, ...). Zero is blank. - auto emit_counter = [&](uint64_t x, int width) { + // When `approx`, the value is a conservative upper bound and gets a '<' + // immediately to its left (consuming one leading pad space). + auto emit_counter = [&](uint64_t x, int width, bool approx = false) { uint64_t target = sstr.size() + width; if (x) { const char *suffixes[] = {" ", "K", "M", "G", "T", "P", "E"}; @@ -604,8 +622,16 @@ WEAK void halide_profiler_report_unlocked(void *user_context, halide_profiler_st scale++; x = (x + 499) / 1000; } + bool reserved = approx; for (uint64_t y = x; y < 10000; y *= 10) { - sstr << " "; + if (reserved) { + reserved = false; // leave room for the '<' + } else { + sstr << " "; + } + } + if (approx) { + sstr << "<"; } sstr << x; target += emit_dim(suffixes[scale]); @@ -615,9 +641,9 @@ WEAK void halide_profiler_report_unlocked(void *user_context, halide_profiler_st // Positive float, up to two decimal places. Falls back to emit_counter // for values that don't fit. - auto emit_float = [&](float x, int width) { + auto emit_float = [&](float x, int width, bool approx = false) { if (x >= 10000) { - emit_counter((uint64_t)x, width); + emit_counter((uint64_t)x, width, approx); return; } uint64_t target = sstr.size() + width; @@ -625,7 +651,10 @@ WEAK void halide_profiler_report_unlocked(void *user_context, halide_profiler_st left_pad += x < 10; left_pad += x < 100; left_pad += x < 1000; - pad_bytes_to(sstr.size() + left_pad); + pad_bytes_to(sstr.size() + left_pad - (approx ? 1 : 0)); + if (approx) { + sstr << "<"; + } sstr << x; pad_bytes_to(target); truncate_bytes_to(target); @@ -633,11 +662,11 @@ WEAK void halide_profiler_report_unlocked(void *user_context, halide_profiler_st // A counter accumulated over `runs` runs. Renders the per-run value if // constant per run, otherwise the average. Zero is blank. - auto emit_normalized_counter = [&](uint64_t x, uint32_t runs, int width) { + auto emit_normalized_counter = [&](uint64_t x, uint32_t runs, int width, bool approx = false) { if (x % runs == 0) { - emit_counter(x / runs, width); + emit_counter(x / runs, width, approx); } else { - emit_float((float)x / runs, width); + emit_float((float)x / runs, width, approx); } }; @@ -658,14 +687,13 @@ WEAK void halide_profiler_report_unlocked(void *user_context, halide_profiler_st constexpr const char *horiz_rule = "--------------------------------------------------------------------------------------------------------\n"; constexpr const char *func_row = - "NNNNNNNNNNNNNNNNNNNNNNNNN|TTTTTTTTT PPPPPPPP|HHHHHH |AAAAAA|MMMMMM|VVVVVV|"; + "NNNNNNNNNNNNNNNNNNNNNNNNN|TTTTTTTTT PPPPPPPP|HHHHHH |LLLLLL|KKKKKK|AAAAAA|MMMMMM|VVVVVV|RRRRRRRR |"; constexpr const char *allocation_func_row = - "NNNNNNNNNNNNNNNNNNNNNNNNN|ZZZZZZZZZZZZZZZZZZ| |AAAAAA|MMMMMM|VVVVVV|"; - // Hand-aligned with func_row above; resize together. + "NNNNNNNNNNNNNNNNNNNNNNNNN|ZZZZZZZZZZZZZZZZZZ| | | |AAAAAA|MMMMMM|VVVVVV| |"; constexpr const char *column_legend_row_1 = - " name | time percent | active| heap | peak | avg |"; + " name | time percent | active| parallel | heap | peak | avg |recompute|"; constexpr const char *column_legend_row_2 = - " | |threads|allocs| mem | mem |"; + " | |threads| loops| tasks|allocs| mem | mem | ratio |"; for (halide_profiler_pipeline_stats *p = s->pipelines; p; p = (halide_profiler_pipeline_stats *)(p->next)) { @@ -673,13 +701,21 @@ WEAK void halide_profiler_report_unlocked(void *user_context, halide_profiler_st continue; } + // Is the pipeline entirely serial? + uint64_t total_parallel_loops = 0; + uint64_t total_parallel_tasks = 0; + for (int i = 0; i < p->num_funcs; i++) { + total_parallel_loops += p->funcs[i].parallel_loops; + total_parallel_tasks += p->funcs[i].parallel_tasks; + } + bool serial = total_parallel_loops == 0; + // Pipeline summary (free-form, not column-aligned). Times are // averaged over billed_runs (runs that produced samples), not // total runs — see halide_profiler_instance_end for why. { float total_ms = p->time / 1000000.0f; int time_runs = p->billed_runs ? p->billed_runs : 1; - float threads = p->active_threads_numerator / (p->active_threads_denominator + 1e-10f); sstr.clear(); emit_dim(horiz_rule); sstr << p->name << "\n" @@ -690,8 +726,14 @@ WEAK void halide_profiler_report_unlocked(void *user_context, halide_profiler_st sstr << " (" << p->billed_runs << " timed)"; } sstr << " time per run: " << total_ms / time_runs << " ms\n"; - if (threads > 1.01f) { - sstr << " average threads used: " << threads << "\n"; + if (!serial) { + float threads = p->active_threads_numerator / (p->active_threads_denominator + 1e-10f); + sstr << " average threads used: " << threads + << " parallel loops: "; + emit_si(total_parallel_loops / p->runs); + sstr << " parallel tasks: "; + emit_si(total_parallel_tasks / p->runs); + sstr << "\n"; } sstr << " heap allocations: " << p->num_allocs << " peak heap usage: "; @@ -724,19 +766,40 @@ WEAK void halide_profiler_report_unlocked(void *user_context, halide_profiler_st __builtin_memset(visited, 0, p->num_funcs * sizeof(bool)); int tree_count = 0; auto dfs = [&](auto &self, int parent_idx, int depth) -> void { + // Emit hoist_storage allocation entries (always leaves) before + // the producer entries at this level. In the IR a Func's + // Allocate precedes its Produce, so this keeps the allocation + // row ahead of the computation it feeds — which compute_with + // otherwise inverts, since it nests one Func's production inside + // a sibling's loop while its storage stays a sibling here. + auto is_alloc = [&](int i) { + return p->funcs[i].kind == halide_profiler_func_kind_allocation; + }; + // The last emitted child (drives └ vs ├): the last producer if + // any, else the last allocation. int last = -1; for (int i = 0; i < p->num_funcs; i++) { - if (p->funcs[i].parent == parent_idx && !visited[i]) { + if (p->funcs[i].parent == parent_idx && !visited[i] && !is_alloc(i)) { last = i; } } - for (int i = 0; i < p->num_funcs; i++) { - if (p->funcs[i].parent == parent_idx && !visited[i]) { - visited[i] = true; - func_depth[i] = depth; - is_last_sibling[i] = (i == last); - tree_order[tree_count++] = i; - self(self, i, depth + 1); + if (last == -1) { + for (int i = 0; i < p->num_funcs; i++) { + if (p->funcs[i].parent == parent_idx && !visited[i]) { + last = i; + } + } + } + for (int pass = 0; pass < 2; pass++) { + for (int i = 0; i < p->num_funcs; i++) { + if (p->funcs[i].parent == parent_idx && !visited[i] && + is_alloc(i) == (pass == 0)) { + visited[i] = true; + func_depth[i] = depth; + is_last_sibling[i] = (i == last); + tree_order[tree_count++] = i; + self(self, i, depth + 1); + } } } }; @@ -758,6 +821,11 @@ WEAK void halide_profiler_report_unlocked(void *user_context, halide_profiler_st // Average threads active for this func and all children uint64_t active_threads_numerator; uint64_t active_threads_denominator; + + // Number of tasks for all containing parallel loops. Note this is + // cumulative in the opposite direction - it incorporates + // information from parents, not children. + uint64_t parallel_tasks; }; size_t cum_stats_size = p->num_funcs * sizeof(CumulativeStats); CumulativeStats *cum_stats = (CumulativeStats *)__builtin_alloca(cum_stats_size); @@ -775,6 +843,20 @@ WEAK void halide_profiler_report_unlocked(void *user_context, halide_profiler_st cum_stats[parent].active_threads_denominator += cum_stats[j].active_threads_denominator; } } + // Propagation to children: parallel_tasks latches downward — a Func + // realized inside its parent's parallel loop "inherits" the parent's + // task count if it doesn't have one of its own. + for (int i = 0; i < p->num_funcs; i++) { + int j = tree_order[i]; + int parent = p->funcs[j].parent; + if (parent >= 0) { + if (p->funcs[j].parallel_tasks == 0) { + cum_stats[j].parallel_tasks = cum_stats[parent].parallel_tasks; + } else { + cum_stats[j].parallel_tasks = p->funcs[j].parallel_tasks; + } + } + } // Rows to print, in tree-DFS order, skipping bookkeeping slots // that would be noise (no time, no allocs). @@ -871,19 +953,46 @@ WEAK void halide_profiler_report_unlocked(void *user_context, halide_profiler_st pad_bytes_to(sstr.size() + w); } break; + case 'L': + emit_normalized_counter(fs->parallel_loops, p->runs, w, + counter_is_approximate(fs, counter_parallel_loops)); + break; + case 'K': + emit_normalized_counter(fs->parallel_tasks, p->runs, w, + counter_is_approximate(fs, counter_parallel_tasks)); + break; case 'A': - emit_normalized_counter(fs->num_allocs, p->runs, w); + emit_normalized_counter(fs->num_allocs, p->runs, w, + counter_is_approximate(fs, counter_num_allocs)); break; case 'M': emit_counter(fs->num_allocs ? fs->memory_peak : fs->stack_peak, w); break; case 'V': if (fs->num_allocs) { - emit_counter(fs->memory_total / fs->num_allocs, w); + emit_counter(fs->memory_total / fs->num_allocs, w, + counter_is_approximate(fs, counter_memory_total)); + } else { + pad_bytes_to(sstr.size() + w); + } + break; + case 'R': { + // points_required_at_root is billed only to the + // canonical instance; look it up there. Use the + // points_computed counter (pure-def stage-0 stores + // by lane count, summed across instances) so the + // ratio reflects what was actually computed, not + // just the realize-box-size machinery. + uint64_t at_root = p->funcs[fs->canonical_id].points_required_at_root; + if (at_root) { + float recompute = (fs->points_computed / (float)at_root); + emit_float(recompute, w, + counter_is_approximate(fs, counter_points_computed)); } else { pad_bytes_to(sstr.size() + w); } break; + } case '|': // Column separator, dimmed so the data stands out. for (int i = 0; i < w; i++) { @@ -1032,6 +1141,7 @@ WEAK void halide_profiler_report_unlocked(void *user_context, halide_profiler_st field_u64(" ", "memory_total", pp->memory_total); field_u64(" ", "active_threads_numerator", pp->active_threads_numerator); field_u64(" ", "active_threads_denominator", pp->active_threads_denominator); + field_u64(" ", "native_vector_bytes", pp->native_vector_bytes); json << " \"funcs\": ["; for (int i = 0; i < pp->num_funcs; i++) { @@ -1043,6 +1153,7 @@ WEAK void halide_profiler_report_unlocked(void *user_context, halide_profiler_st field_i(" ", "canonical_id", fs->canonical_id); field_i(" ", "kind", fs->kind); field_i(" ", "buffer_func_id", fs->buffer_func_id); + field_u64(" ", "counters_approximated", fs->counters_approximated); field_u64(" ", "time_ns", fs->time); field_u64(" ", "memory_current", fs->memory_current); field_u64(" ", "memory_peak", fs->memory_peak); @@ -1050,7 +1161,11 @@ WEAK void halide_profiler_report_unlocked(void *user_context, halide_profiler_st field_u64(" ", "stack_peak", fs->stack_peak); field_u64(" ", "active_threads_numerator", fs->active_threads_numerator); field_u64(" ", "active_threads_denominator", fs->active_threads_denominator); - field_u64(" ", "num_allocs", fs->num_allocs, true); + field_u64(" ", "num_allocs", fs->num_allocs); + field_u64(" ", "parallel_loops", fs->parallel_loops); + field_u64(" ", "parallel_tasks", fs->parallel_tasks); + field_u64(" ", "points_required_at_root", fs->points_required_at_root); + field_u64(" ", "points_computed", fs->points_computed, true); json << " }"; // Flush periodically so we don't overflow the buffer for diff --git a/src/runtime/profiler_inlined.cpp b/src/runtime/profiler_inlined.cpp index 217a72493288..3c59557f9e3f 100644 --- a/src/runtime/profiler_inlined.cpp +++ b/src/runtime/profiler_inlined.cpp @@ -60,4 +60,44 @@ WEAK_INLINE int halide_profiler_decr_active_threads(halide_profiler_instance_sta return atomic_fetch_sub_sequentially_consistent(&(instance->active_threads), 1); } + +WEAK_INLINE int halide_profiler_update_counters(struct halide_profiler_instance_state *instance, + int id, + uint64_t memory_total, + uint64_t num_allocs, + uint64_t parallel_loops, + uint64_t parallel_tasks, + uint64_t points_required_at_root, + uint64_t points_computed) { + using namespace Halide::Runtime::Internal::Synchronization; + + halide_profiler_func_stats &stats = instance->funcs[id]; + + // This gets inlined. If this is in an inner loop, most of the args will be + // the constant zero. We therefore test for zero before adding to every + // counter so that unused counters compile to no code. +#define UPDATE_COUNTER(X) \ + if (X) { \ + atomic_fetch_add_sequentially_consistent(&(stats.X), X); \ + } + + UPDATE_COUNTER(memory_total); + UPDATE_COUNTER(num_allocs); + UPDATE_COUNTER(parallel_loops); + UPDATE_COUNTER(parallel_tasks); + UPDATE_COUNTER(points_required_at_root); + UPDATE_COUNTER(points_computed); + +#undef UPDATE_COUNTER + + // Mirror memory_total and num_allocs at the instance level. + if (memory_total) { + atomic_add_fetch_sequentially_consistent(&instance->memory_total, memory_total); + } + if (num_allocs) { + atomic_add_fetch_sequentially_consistent(&instance->num_allocs, (int)num_allocs); + } + + return 0; +} } diff --git a/src/runtime/runtime_internal.h b/src/runtime/runtime_internal.h index b874e448c7c9..fdf951c7dc87 100644 --- a/src/runtime/runtime_internal.h +++ b/src/runtime/runtime_internal.h @@ -179,6 +179,8 @@ WEAK int halide_profiler_instance_start(void *user_context, const int *func_canonical_ids, const int *func_kinds, const int *func_buffer_func_ids, + const uint32_t *func_counters_approximated, + uint64_t native_vector_bytes, halide_profiler_instance_state *instance); WEAK int halide_profiler_instance_end(void *user_context, halide_profiler_instance_state *instance); diff --git a/test/correctness/simplify.cpp b/test/correctness/simplify.cpp index f9811297b482..ab084c540c65 100644 --- a/test/correctness/simplify.cpp +++ b/test/correctness/simplify.cpp @@ -1493,6 +1493,12 @@ void check_boolean() { check(select(x > 5, 2, 3) + select(x > 5, 6, 2), select(5 < x, 8, 5)); check(select(x > 5, 8, 3) - select(x > 5, 6, 2), select(5 < x, 2, 1)); + // A comparison of a constant against a select of constants folds for + // any type, including unsigned ones (which don't satisfy no_overflow). + check(0 < select(x < 5, 4, 0), x < 5); + check(make_zero(UInt(32)) < select(x < 5, make_const(UInt(32), 4), make_const(UInt(32), 0)), x < 5); + check(select(x < 5, make_const(UInt(32), 0), make_const(UInt(32), 4)) < make_const(UInt(32), 1), x < 5); + check(select(x < 5, select(x < 5, 0, 1), 2), select(x < 5, 0, 2)); check(select(x < 5, 0, select(x < 5, 1, 2)), select(x < 5, 0, 2)); diff --git a/test/generator/profiler_instances_aottest.cpp b/test/generator/profiler_instances_aottest.cpp index 6cac9c15fcf1..816f367b4f69 100644 --- a/test/generator/profiler_instances_aottest.cpp +++ b/test/generator/profiler_instances_aottest.cpp @@ -69,6 +69,69 @@ void check_unscheduled_update_multiple_entries(const halide_profiler_pipeline_st REQUIRE(fs[0]->canonical_id == fs[1]->canonical_id); } +// RoundUp tail strategy on a compute_root Func over-computes the tail +// of the loop. The realize-box counter (points_required_at_realization) +// reflects the requested extent and misses this; points_computed +// (pure-def stage-0 stores × lanes) catches it. So we expect +// points_computed to exceed points_required_at_root. +void check_roundup_overstores_bytes(const halide_profiler_pipeline_stats *p) { + auto fs = entries_of(p, "roundup_outer"); + REQUIRE(fs.size() == 1); + REQUIRE(fs[0]->points_required_at_root > 0); + REQUIRE(fs[0]->points_computed > fs[0]->points_required_at_root); +} + +// GuardWithIf tail strategy: the tail iterations are guarded, so no +// extra stores actually happen. points_computed matches the required +// extent. +void check_guardwithif_no_overstore(const halide_profiler_pipeline_stats *p) { + auto fs = entries_of(p, "guard_outer"); + REQUIRE(fs.size() == 1); + REQUIRE(fs[0]->points_required_at_root > 0); + REQUIRE(fs[0]->points_computed == fs[0]->points_required_at_root); +} + +// Fully-unrolled Func with both a pure def and an update def. There are +// no stage-naming For loops in the IR — only the declare_stage marker +// from ScheduleFunctions can distinguish pure-def stores from update- +// def stores. points_computed should count exactly the pure-def +// stores (4 of them, one per unrolled iteration), not the update-def +// stores. +void check_unrolled_pure_update(const halide_profiler_pipeline_stats *p) { + auto fs = entries_of(p, "unrolled_pu"); + REQUIRE(fs.size() == 1); + REQUIRE(fs[0]->points_computed == 4); +} + +// compute_with: two Funcs share a loop nest, so their stage stores +// appear interleaved in the IR. The per-Func pure-def tracking has to +// attribute each Store to the right Func. compute_with also gives each +// Func multiple entries (one for the actual producer plus +// box-required artifacts under the fused partner), so we sum across +// entries. Each Func is pure-only and gets stored at every output +// point exactly once, so its points_computed should equal its +// points_required_at_root. +void check_compute_with(const halide_profiler_pipeline_stats *p) { + auto a = entries_of(p, "cw_a"); + auto b = entries_of(p, "cw_b"); + REQUIRE(!a.empty()); + REQUIRE(!b.empty()); + uint64_t a_computed = 0; + uint64_t b_computed = 0; + for (auto *fs : a) { + a_computed += fs->points_computed; + } + for (auto *fs : b) { + b_computed += fs->points_computed; + } + int a_canon = a[0]->canonical_id; + int b_canon = b[0]->canonical_id; + REQUIRE(a_computed > 0); + REQUIRE(b_computed > 0); + REQUIRE(a_computed == p->funcs[a_canon].points_required_at_root); + REQUIRE(b_computed == p->funcs[b_canon].points_required_at_root); +} + // GPU-only: an outer CPU loop with a host-then-device-then-host data // chain forces explicit halide_copy_to_host / halide_copy_to_device calls // to fire once per outer iteration. The synthetic copy "Func" entries @@ -122,6 +185,171 @@ void check_mixed_host_device_update_defs(const halide_profiler_pipeline_stats *p REQUIRE(found_mid_func_copy); } +// tab is an inlined Func whose root box is `ux * ux` for +// ux = cast(cast(x)) — bounds inference can't prove the +// product fits in int32 ([0, 65535] * [0, 65535] = up to 4_294_836_225, +// which overflows), so simplify materialises a signed_integer_overflow +// intrinsic inside the declare_box_required_at_root marker for tab. +// (compute_root'ing tab with this same index expression makes the same +// intrinsic reach codegen and user_errors.) Without the poison-drop +// pre-pass in inject_profiling that marker reaches codegen and breaks +// the compile; with the pre-pass the marker is silently dropped, the +// pipeline compiles, and tab's points_required_at_root counter stays at +// zero (we lose the root-box count for the poisoned chain but +// everything else still works). tab_caller, the inlined wrapper that +// consumes tab, still has a well-defined root box of its own. +void check_points_required_at_root_canonical_only(const halide_profiler_pipeline_stats *p) { + // For any Func with multiple entries, at most one entry should have a + // non-zero points_required_at_root (the canonical one — that's where the + // compiler bills the pipeline-wide root box). + int entries_with_pr_at_root = 0; + int total_multi_entry_funcs = 0; + auto check = [&](const char *name) { + auto xs = entries_of(p, name); + if (xs.size() <= 1) { + return; + } + total_multi_entry_funcs++; + int with_pr = 0; + int canon = xs[0]->canonical_id; + for (auto *fs : xs) { + if (fs->points_required_at_root > 0) { + with_pr++; + REQUIRE((int)(fs - p->funcs) == canon); + } + } + REQUIRE(with_pr <= 1); + entries_with_pr_at_root += with_pr; + }; + check("update_f"); + REQUIRE(total_multi_entry_funcs >= 1); +} + +// GPU-only: block-level producers stored in GPU shared / global (heap) +// memory. FuseGPUThreadLoops hoists their per-Func allocations out of the +// thread loops and coalesces them into one backing allocation. Before +// per-Func allocation naming this fused allocation showed up in the +// profiler as an orphan allocate row owned by no Func, and the per-Func +// allocation bytes were lost. Now each producer keeps its own name and is +// billed its own allocation size, so shared_a, shared_b, and shared_heap_h +// each get a single Func entry, parented under shared_out, reporting a +// non-zero memory_total and num_allocs. +void check_within_block_gpu_allocations_attributed(const halide_profiler_pipeline_stats *p) { + auto shared_out = entries_of(p, "shared_out"); + REQUIRE(shared_out.size() == 1); + int shared_out_id = (int)(shared_out[0] - p->funcs); + + auto descends_from = [&](int idx, int ancestor_id) { + while (idx >= 0) { + if (idx == ancestor_id) { + return true; + } + idx = p->funcs[idx].parent; + } + return false; + }; + + for (const char *name : {"shared_a", "shared_b", "shared_heap_h"}) { + auto fs = entries_of(p, name); + REQUIRE(fs.size() == 1); + // A real Func entry, not a synthetic allocation/copy row. + REQUIRE(fs[0]->kind == halide_profiler_func_kind_func); + // The within-block allocation was billed to this Func. + REQUIRE(fs[0]->num_allocs > 0); + REQUIRE(fs[0]->memory_total > 0); + // Sensible size: at least one byte per recorded allocation. + REQUIRE(fs[0]->memory_total >= fs[0]->num_allocs); + // Parented inside the shared_out producer tree, not orphaned at root. + int idx = (int)(fs[0] - p->funcs); + REQUIRE(descends_from(idx, shared_out_id)); + } +} + +// GPU points_computed for the within-block producers. Keeping per-Func +// store names (the fix under test) is what lets the stage-0 store counter +// reach each Func at all; before it, these stores hung off the fused +// backing-allocation name and were mis- or un-attributed. On GPU the count +// is a conservative upper bound rather than an exact tally: FuseGPUThreadLoops +// fuses the block's producers into one thread loop sized to the largest +// footprint and guards each producer's stores to its own footprint, but the +// profiler can't flush counters mid-kernel, so it hoists each per-thread +// contribution out by its loop-var upper bound (Profiling.cpp's +// hoist_loop_var_upper_bound) and scales by the fused thread extent. That is +// exact for the producer filling the thread extent and an over-estimate for +// the narrower ones. What must always hold: the store attribution reaches +// each Func (non-zero) and never under-counts its root footprint. (On CPU +// these Funcs count exactly — points_computed == points_required_at_root +// with no recompute — verified out of band.) +void check_within_block_gpu_points_computed(const halide_profiler_pipeline_stats *p) { + for (const char *name : {"shared_a", "shared_b", "shared_heap_h"}) { + auto fs = entries_of(p, name); + REQUIRE(fs.size() == 1); + uint64_t at_root = p->funcs[fs[0]->canonical_id].points_required_at_root; + REQUIRE(at_root > 0); + REQUIRE(fs[0]->points_computed > 0); + REQUIRE(fs[0]->points_computed >= at_root); + } +} + +// GPU-only: a compute_root Func (dev_only_mid) consumed only on the device. +// InjectHostDevBufferCopies nulls its host allocation because the buffer +// lives solely in device global memory, so the profiler — which tracks +// memory at the host Allocate — would see a zero-sized allocation and bill +// nothing. IHDBC emits a declare_allocation marker carrying the device +// buffer's byte size at the null-out site. InjectCounters bills it to +// num_allocs/memory_total, and InjectProfiling turns it into a matched +// memory_allocate/memory_free pair (the host Free node still brackets the +// device lifetime) so memory_peak/current are tracked too. Assert +// dev_only_mid gets a single Func entry with non-zero num_allocs, +// memory_total, and memory_peak. +void check_device_only_compute_root_allocation(const halide_profiler_pipeline_stats *p) { + auto fs = entries_of(p, "dev_only_mid"); + REQUIRE(fs.size() == 1); + REQUIRE(fs[0]->kind == halide_profiler_func_kind_func); + REQUIRE(fs[0]->num_allocs > 0); + REQUIRE(fs[0]->memory_total > 0); + REQUIRE(fs[0]->memory_total >= fs[0]->num_allocs); + // The device allocation's lifetime is tracked (matched allocate/free), + // so its peak is billed. + REQUIRE(fs[0]->memory_peak > 0); +} + +// The fused backing allocation that FuseGPUThreadLoops emits for coalesced +// within-block GPU allocations carries a synthetic name (unique_name of +// "shared_alloc" / "global_alloc", historically "allocgroup__f1__f2..." +// rendered with commas). Such a name corresponds to no Func, so if it ever +// reaches the profiler as its own entry it is an orphan allocate row. Assert +// that no entry carries one of these synthetic names. +void check_no_orphan_allocation_entries(const halide_profiler_pipeline_stats *p) { + for (int i = 0; i < p->num_funcs; i++) { + const char *name = p->funcs[i].name; + REQUIRE(strncmp(name, "shared_alloc", strlen("shared_alloc")) != 0); + REQUIRE(strncmp(name, "global_alloc", strlen("global_alloc")) != 0); + REQUIRE(strstr(name, "allocgroup") == nullptr); + REQUIRE(strchr(name, ',') == nullptr); + } +} + +// The counters_approximated bitmask flags counters that are conservative +// upper bounds rather than exact. It only happens on GPU, where a guarded +// contribution summed over a loop can't be counted exactly (the reporter +// marks those columns with a leading '<'). On CPU every counter is flushed +// at runtime, so nothing is ever flagged. Verify the mechanism is active on +// GPU (at least one entry flagged) and silent on CPU. +void check_counters_approximated(const halide_profiler_pipeline_stats *p, bool has_gpu) { + int flagged = 0; + for (int i = 0; i < p->num_funcs; i++) { + if (p->funcs[i].counters_approximated) { + flagged++; + } + } + if (has_gpu) { + REQUIRE(flagged > 0); + } else { + REQUIRE(flagged == 0); + } +} + } // namespace int main(int argc, char **argv) { @@ -146,6 +374,11 @@ int main(int argc, char **argv) { check_two_compute_root_callers(target); check_unscheduled_update_multiple_entries(target); + check_roundup_overstores_bytes(target); + check_guardwithif_no_overstore(target); + check_unrolled_pure_update(target); + check_compute_with(target); + // Only present when the pipeline was built with a GPU feature — the // generator gates the corresponding Funcs on get_target().has_gpu_feature(). if (!entries_of(target, "xfer_out").empty()) { @@ -154,6 +387,21 @@ int main(int argc, char **argv) { if (!entries_of(target, "mixed_sched").empty()) { check_mixed_host_device_update_defs(target); } + if (!entries_of(target, "shared_out").empty()) { + check_within_block_gpu_allocations_attributed(target); + check_within_block_gpu_points_computed(target); + } + if (!entries_of(target, "dev_only_mid").empty()) { + check_device_only_compute_root_allocation(target); + } + + check_counters_approximated(target, !entries_of(target, "shared_out").empty()); + + // Holds regardless of target: the fused-allocation backing name should + // never surface as its own profiler entry. + check_no_orphan_allocation_entries(target); + + check_points_required_at_root_canonical_only(target); printf("Success!\n"); return 0; diff --git a/test/generator/profiler_instances_generator.cpp b/test/generator/profiler_instances_generator.cpp index 9bd95a9d49b6..10a6cc39dd13 100644 --- a/test/generator/profiler_instances_generator.cpp +++ b/test/generator/profiler_instances_generator.cpp @@ -198,6 +198,51 @@ class ProfilerInstances : public Generator { mixed_sched.update(1).gpu_tile(x, xi, 8); } + // GPU-only: within-block GPU allocations. A chain of stencil + // producers computed at the block level and stored in GPU shared + // and global memory. FuseGPUThreadLoops hoists their per-Func + // allocations out of the thread loops and coalesces them into a + // single backing allocation with a synthetic name. Before per-Func + // allocation naming, that fused allocation surfaced in the profiler + // as an orphan allocate row owned by no Func (and the per-Func bytes were + // lost). Now each producer keeps its own name and is billed its own + // allocation size, so its entry reports a non-zero memory_total and + // there is no orphan row for the backing allocation. shared_heap_h + // sits inside the thread loop and is dynamically sized, so it lands + // in per-block global (heap) memory rather than shared — the other + // half of the fused-allocation path. + Func shared_a("shared_a"), shared_b("shared_b"), + shared_heap_h("shared_heap_h"), shared_out("shared_out"); + if (get_target().has_gpu_feature()) { + shared_a(x) = x + 1; + shared_b(x) = shared_a(x) + shared_a(x + 1); + shared_heap_h(x) = shared_b(x) + shared_b(x + 1); + shared_out(x) = shared_heap_h(x); + Var xo, xi; + shared_out.compute_root().gpu_tile(x, xo, xi, 16); + shared_a.compute_at(shared_out, xo).gpu_threads(x).store_in(MemoryType::GPUShared); + shared_b.compute_at(shared_out, xo).gpu_threads(x).store_in(MemoryType::GPUShared); + shared_heap_h.compute_at(shared_out, xi).store_in(MemoryType::Heap); + } + + // GPU-only: a compute_root Func consumed only on the device. + // dev_only_mid is produced on the device and read only by + // dev_only_out, also on the device, so InjectHostDevBufferCopies + // sees it touched on a single device and nulls its host allocation + // (the data lives in device global memory). The profiler tracks + // memory at the host Allocate, which then carries no size — so + // without a device-allocation marker dev_only_mid would report zero + // memory. IHDBC emits a declare_allocation at the null-out site; we + // assert dev_only_mid's device buffer is billed a non-zero size. + Func dev_only_mid("dev_only_mid"), dev_only_out("dev_only_out"); + if (get_target().has_gpu_feature()) { + dev_only_mid(x) = x * 2; + dev_only_out(x) = dev_only_mid(x) + dev_only_mid(x + 1); + Var xi; + dev_only_mid.compute_root().gpu_tile(x, xi, 32); + dev_only_out.compute_root().gpu_tile(x, xi, 32); + } + // Extern stage. Func extern_stage_e("extern_stage_e"); extern_stage_e.define_extern("test_extern_stage", @@ -244,7 +289,8 @@ class ProfilerInstances : public Generator { slide_out(x) + slide_fail_f(x) + extern_stage_e(x) + inwards_red_root(x) + inwards_red_at_y(x); if (get_target().has_gpu_feature()) { - out_value = out_value + approx_out(x) + xfer_out(x) + mixed_sched(x); + out_value = out_value + approx_out(x) + xfer_out(x) + mixed_sched(x) + + shared_out(x) + dev_only_out(x); } out(x) = out_value;