Skip to content

fix(pc-sampling): count _not_issued stall samples separately #166

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
CodingInAVan merged 1 commit into from
Sep 30, 2026
Merged

fix(pc-sampling): count _not_issued stall samples separately #166

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 21 additions & 5 deletions include/gpufl/backends/nvidia/engine/pc_sampling_engine.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,16 @@ constexpr int kMaxGetDataCalls = 1024;
// Rows per Monitor::PushProfileSamples call, bounding the staging vector.
constexpr size_t kRowsPerPush = 8192;

// CUPTI counts each sample under its warp state and, when the scheduler
// issued nothing that cycle, again under the state's _not_issued twin, so
// only the other reasons add up to totalSamples.
bool IsNotIssuedReason(const std::string& name) {
static const std::string kSuffix = "_not_issued";
return name.size() > kSuffix.size() &&
name.compare(name.size() - kSuffix.size(), kSuffix.size(),
kSuffix) == 0;
}

PCSamplingBuffers* AllocatePcSamplingBuffers(const size_t numPcs,
const size_t stallSlots) {
auto* b = new PCSamplingBuffers();
Expand Down Expand Up @@ -867,6 +877,7 @@ void PcSamplingEngine::CollectPcSamplingData_() {
std::vector<ProfileSampleInput> rows;
size_t rowsEmitted = 0;
uint64_t samplesEmitted = 0;
uint64_t notIssuedEmitted = 0;
auto pushRows = [&rows, &rowsEmitted] {
if (rows.empty()) return;
Monitor::PushProfileSamples(rows);
Expand Down Expand Up @@ -931,6 +942,7 @@ void PcSamplingEngine::CollectPcSamplingData_() {
" nonUsrKernelsTotalSamples=", nonUsrSamples);

uint64_t samplesThisCall = 0;
uint64_t notIssuedThisCall = 0;
for (size_t i = 0; i < numPcs; ++i) {
const CUpti_PCSamplingPCData& pc = batch->pPcData[i];
if (pc.stallReasonCount == 0 || !pc.stallReason) continue;
Expand All @@ -953,7 +965,10 @@ void PcSamplingEngine::CollectPcSamplingData_() {
const uint32_t samples = pc.stallReason[j].samples;
const uint32_t reason =
pc.stallReason[j].pcSamplingStallReasonIndex;
samplesThisCall += samples;
const auto name = reasonNames.find(reason);
const bool notIssued = name != reasonNames.end() &&
IsNotIssuedReason(name->second);
(notIssued ? notIssuedThisCall : samplesThisCall) += samples;
if (samples == 0) continue;
if (!deviceIdKnown) {
deviceIdKnown = true;
Expand All @@ -970,7 +985,6 @@ void PcSamplingEngine::CollectPcSamplingData_() {
s.device_id = deviceId;
s.function_key = source->second.functionKey;
s.pc_offset = static_cast<uint32_t>(pcOffset);
const auto name = reasonNames.find(reason);
s.metric_name = name != reasonNames.end()
? name->second
: "Stall_" + std::to_string(reason);
Expand All @@ -980,14 +994,15 @@ void PcSamplingEngine::CollectPcSamplingData_() {
s.source_file = source->second.sourceFile;
s.source_line = source->second.sourceLine;
rows.push_back(std::move(s));
samplesEmitted += samples;
(notIssued ? notIssuedEmitted : samplesEmitted) += samples;
}
if (rows.size() >= kRowsPerPush) pushRows();
}
pushRows();

GFL_LOG_DEBUG("[PC Sampling] GetData returned ", samplesThisCall,
" samples across ", numPcs, " PC records");
" samples (+", notIssuedThisCall, " not issued) across ",
numPcs, " PC records");
// Drained once a call returns nothing and CUPTI reports nothing
// pending. Two empty calls in a row with something still pending end
// it too, so a stuck report cannot spin.
Expand All @@ -1002,7 +1017,8 @@ void PcSamplingEngine::CollectPcSamplingData_() {
}

GFL_LOG_DEBUG("[PC Sampling] collect summary: ", rowsEmitted, " rows, ",
samplesEmitted, " samples across ", sourceByPc.size(),
samplesEmitted, " samples (+", notIssuedEmitted,
" not issued) across ", sourceByPc.size(),
" PCs; totalSamples=", sumTotal, " dropped=", sumDropped,
" nonUsrKernels=", sumNonUsr,
hardwareBufferFull ? " (hardware buffer overflowed)" : "");
Expand Down
4 changes: 4 additions & 0 deletions include/gpufl/report/hint_engine.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,10 @@ namespace report {
struct FuncProfile {
std::map<std::string, uint64_t> stalls; // display-name → count
uint64_t totalStalls = 0;
// Samples also counted in `stalls`, taken on cycles where the scheduler
// issued no instruction (CUPTI's `_not_issued` reasons).
std::map<std::string, uint64_t> notIssuedStalls;
uint64_t totalNotIssued = 0;
uint64_t warpInsts = 0;
uint64_t threadInsts = 0;
uint64_t globalSectors = 0;
Expand Down
61 changes: 38 additions & 23 deletions include/gpufl/report/text_report.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1550,31 +1550,30 @@ void TextReport::writeProfileAnalysis(std::ostringstream& out) const {
return;
}

// CUPTI counts each sample under its warp state and, when the scheduler
// issued nothing that cycle, again under the state's _not_issued twin.
const std::string notIssued = "_not_issued";
auto isNotIssued = [&notIssued](const std::string& raw) {
return raw.size() > notIssued.size() &&
raw.compare(raw.size() - notIssued.size(), notIssued.size(),
notIssued) == 0;
};

// Convert a raw CUPTI stall metric name to a human-readable short name.
// e.g. "smsp__pcsamp_warps_issue_stalled_wait_not_issued" → "Wait (not issued)"
auto shortenStallName = [](const std::string& raw) -> std::string {
// e.g. "smsp__pcsamp_warps_issue_stalled_wait_not_issued" → "Wait"
auto shortenStallName = [&](const std::string& raw) -> std::string {
const std::string prefix = "smsp__pcsamp_warps_issue_stalled_";
std::string s = raw;
if (s.size() > prefix.size() && s.substr(0, prefix.size()) == prefix)
s = s.substr(prefix.size());

// Handle "_not_issued" suffix
const std::string notIssued = "_not_issued";
bool isNotIssued = false;
if (s.size() > notIssued.size() &&
s.substr(s.size() - notIssued.size()) == notIssued) {
s = s.substr(0, s.size() - notIssued.size());
isNotIssued = true;
}
if (isNotIssued(s)) s.resize(s.size() - notIssued.size());

// Replace underscores with spaces and capitalize first letter of each word
for (size_t i = 0; i < s.size(); ++i) {
if (s[i] == '_') s[i] = ' ';
if (i == 0 || (i > 0 && s[i-1] == ' '))
s[i] = static_cast<char>(std::toupper(static_cast<unsigned char>(s[i])));
}

if (isNotIssued) s += " (idle)";
return s;
};

Expand All @@ -1601,8 +1600,14 @@ void TextReport::writeProfileAnalysis(std::ostringstream& out) const {

if (ps.stall_reason > 1) {
std::string reason = resolveStallDisplay(ps);
fp.stalls[reason] += ps.metric_value;
fp.totalStalls += ps.metric_value;
if (isNotIssued(ps.reason_name.empty() ? ps.metric_name
: ps.reason_name)) {
fp.notIssuedStalls[reason] += ps.metric_value;
fp.totalNotIssued += ps.metric_value;
} else {
fp.stalls[reason] += ps.metric_value;
fp.totalStalls += ps.metric_value;
}
}
if (ps.metric_name == "smsp__sass_inst_executed")
fp.warpInsts += ps.metric_value;
Expand All @@ -1627,6 +1632,18 @@ void TextReport::writeProfileAnalysis(std::ostringstream& out) const {
if (static_cast<int>(ranked.size()) > top_n_)
ranked.resize(top_n_);

auto writeStallShares = [&out](const std::map<std::string, uint64_t>& counts,
uint64_t total) {
auto stallRanked = sortedTopN(counts, 0, [](uint64_t v) { return static_cast<double>(v); });
for (const auto& [reason, count] : stallRanked) {
double pct = total > 0 ? count * 100.0 / total : 0;
out << " " << std::left << std::setw(28) << truncate(reason, 26)
<< std::right << std::setw(8) << fmtCount(count)
<< std::setw(7) << std::fixed << std::setprecision(1) << pct << "% "
<< makeBar(pct) << "\n";
}
};

// ── Write per-function analysis ─────────────────────────────────────────
for (const auto& [fn, fp] : ranked) {
std::string shortName = shortenKernelName(fn);
Expand All @@ -1637,15 +1654,13 @@ void TextReport::writeProfileAnalysis(std::ostringstream& out) const {

// Stall distribution
if (!fp->stalls.empty()) {
auto stallRanked = sortedTopN(fp->stalls, 0, [](uint64_t v) { return static_cast<double>(v); });
out << " Stalls:\n";
for (const auto& [reason, count] : stallRanked) {
double pct = fp->totalStalls > 0 ? count * 100.0 / fp->totalStalls : 0;
out << " " << std::left << std::setw(28) << truncate(reason, 26)
<< std::right << std::setw(8) << fmtCount(count)
<< std::setw(7) << std::fixed << std::setprecision(1) << pct << "% "
<< makeBar(pct) << "\n";
}
writeStallShares(fp->stalls, fp->totalStalls);
}
if (!fp->notIssuedStalls.empty()) {
out << " Not issued (" << fmtCount(fp->totalNotIssued)
<< " samples on cycles with no instruction issued):\n";
writeStallShares(fp->notIssuedStalls, fp->totalNotIssued);
}

// Instruction analysis
Expand Down
97 changes: 81 additions & 16 deletions python/gpufl/analyzer/analyzer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,8 @@ def _shorten_kernel_name(name: str) -> tuple[str, str]:
return short_func, name


# CUPTI CUpti_ActivityPCSamplingStallReason - skip 0 (invalid) and 1 (none)
# CUPTI CUpti_ActivityPCSamplingStallReason - skip 0 (invalid) and 1 (none).
# Only Activity API rows use these indices; PC Sampling API rows carry names.
_STALL_NAMES: dict[int, str] = {
2: "Instruction Fetch",
3: "Execution Dependency",
Expand All @@ -55,12 +56,37 @@ def _shorten_kernel_name(name: str) -> tuple[str, str]:
7: "Constant Memory",
8: "Pipe Busy",
9: "Memory Throttle",
10: "Branch Resolving",
11: "Wait",
12: "Barrier",
13: "Sleeping",
10: "Not Selected",
11: "Other",
12: "Sleeping",
}

_PC_STALL_PREFIX = "smsp__pcsamp_warps_issue_stalled_"
_NOT_ISSUED_SUFFIX = "_not_issued"


def _pc_stall_reason(metric_name, stall_index) -> tuple:
"""Return (reason, not_issued) for a pc_sampling row.

The PC Sampling API counts each sample under its warp state and, when
the scheduler issued nothing that cycle, again under the state's
``_not_issued`` twin, so the two need separate denominators. Other
``smsp__pcsamp_`` counters (sample_count, samples_data_dropped) are not
warp states. Activity API rows have no name, only an index.
"""
if metric_name:
if metric_name.startswith(_PC_STALL_PREFIX):
reason = metric_name[len(_PC_STALL_PREFIX):]
if reason.endswith(_NOT_ISSUED_SUFFIX):
return reason[:-len(_NOT_ISSUED_SUFFIX)], True
return reason, False
if metric_name.startswith("smsp__pcsamp_"):
return None, False
return metric_name, False
if stall_index > 1:
return _STALL_NAMES.get(stall_index, f"Stall_{stall_index}"), False
return None, False


class GpuFlightSession:
def __init__(
Expand Down Expand Up @@ -535,6 +561,9 @@ def _expand_batches(self, device_df, scope_df, system_df, dict_maps) -> tuple:
scope_name = dict_maps['scope_name'].get(sn_id) if sn_id else None
stall = row[ci['stall_reason']]
mv = row[ci['metric_value']]
reason, not_issued = (
_pc_stall_reason(metric_name, stall)
if sample_kind == 'pc_sampling' else (None, False))
sample_rows.append({
'type': 'profile_sample',
'session_id': batch.get('session_id'),
Expand All @@ -549,7 +578,8 @@ def _expand_batches(self, device_df, scope_df, system_df, dict_maps) -> tuple:
'sample_kind': sample_kind,
'scope_name': scope_name,
# Compatibility aliases used by inspect_stalls / inspect_profile_samples
'reason_name': _STALL_NAMES.get(stall, f"Stall_{stall}") if stall > 1 else None,
'reason_name': reason,
'not_issued': not_issued,
'sample_count': mv if sample_kind == 'pc_sampling' else 0,
})
scopes_df = pd.DataFrame(sample_rows)
Expand Down Expand Up @@ -1080,13 +1110,17 @@ def safe_mode(x):

self.console.print(table)

def inspect_stalls(self, top_n: int = 10):
def inspect_stalls(self, top_n: int = 10, not_issued: bool = False):
"""Show per-kernel stall distribution from PC-sampling data.

Requires ``enablePCSampling=true`` at session init. Joins
``profile_sample`` events to kernels via ``corr_id``, then pivots by
``reason_name`` to show what fraction of samples each stall category
accounts for in the hottest kernels.

Shares are of each kernel's samples, which add up to CUPTI's sample
count. ``not_issued=True`` shows the ``_not_issued`` samples instead:
the subset taken on cycles where the scheduler issued nothing.
"""
if self.scopes.empty or 'type' not in self.scopes.columns:
self.console.print("[yellow]No PC sampling data found - enable PC sampling at session init.[/yellow]")
Expand All @@ -1101,12 +1135,16 @@ def inspect_stalls(self, top_n: int = 10):
self.console.print("[yellow]No profile_sample events found - enable PC sampling at init.[/yellow]")
return

required = {'corr_id', 'reason_name', 'sample_count'}
required = {'corr_id', 'reason_name', 'sample_count', 'not_issued'}
if not required.issubset(samples.columns):
self.console.print(f"[yellow]profile_sample records missing columns: {required - set(samples.columns)}[/yellow]")
return

samples['sample_count'] = pd.to_numeric(samples['sample_count'], errors='coerce').fillna(0)
samples = samples[samples['not_issued'] == not_issued]
if samples.empty:
self.console.print("[yellow]No not-issued samples found.[/yellow]")
return

# Aggregate sample counts: (corr_id, reason_name) → total samples
stall_agg = (
Expand Down Expand Up @@ -1139,7 +1177,8 @@ def inspect_stalls(self, top_n: int = 10):

stall_cols = [c for c in pivot.columns if c not in ('name', 'total_samples')]

table = Table(title=f"Stall Distribution - Top {top_n} Kernels (PC Sampling)")
family = "PC Sampling, not issued" if not_issued else "PC Sampling"
table = Table(title=f"Stall Distribution - Top {top_n} Kernels ({family})")
table.add_column("Kernel", style="cyan", no_wrap=False)
table.add_column("Samples", justify="right")
for col in stall_cols:
Expand Down Expand Up @@ -1205,22 +1244,41 @@ def inspect_profile_samples(self, top_n: int = 10):
pc_samples = pd.DataFrame()

if not pc_samples.empty:
if 'not_issued' not in pc_samples.columns:
pc_samples['not_issued'] = False
# A _not_issued sample is also counted under its warp state, so
# each family gets its own denominator.
has_reason = pc_samples['reason_name'].notna()
states = pc_samples[has_reason & ~pc_samples['not_issued']]
twins = pc_samples[has_reason & pc_samples['not_issued']]
by_reason = (
pc_samples.groupby('reason_name', dropna=False)['sample_count']
states.groupby('reason_name')['sample_count']
.sum()
.sort_values(ascending=False)
.head(top_n)
)
twin_by_reason = twins.groupby('reason_name')['sample_count'].sum()

reason_table = Table(title=f"PC Sampling Reasons - Top {top_n}")
reason_table = Table(
title=f"PC Sampling Reasons - Top {top_n}",
caption="Not issued: samples taken on cycles where the "
"scheduler issued nothing (a subset of Samples).",
)
reason_table.add_column("Reason", style="cyan")
reason_table.add_column("Samples", justify="right")
total_samples = float(pc_samples['sample_count'].sum()) or 1.0
reason_table.add_column("Share", justify="right")
reason_table.add_column("Not issued", justify="right")
reason_table.add_column("Share", justify="right")
total_samples = float(states['sample_count'].sum()) or 1.0
total_twins = float(twins['sample_count'].sum()) or 1.0

for reason, count in by_reason.items():
label = str(reason) if pd.notna(reason) and str(reason) else "unknown"
reason_table.add_row(label, str(int(count)), f"{(count/total_samples)*100:.1f}%")
twin = float(twin_by_reason.get(reason, 0))
reason_table.add_row(
str(reason), str(int(count)),
f"{(count/total_samples)*100:.1f}%",
str(int(twin)), f"{(twin/total_twins)*100:.1f}%",
)
self.console.print(reason_table)

if not self.kernels.empty and 'corr_id' in self.kernels.columns and 'corr_id' in pc_samples.columns:
Expand All @@ -1234,7 +1292,12 @@ def inspect_profile_samples(self, top_n: int = 10):
)

corr_to_name, fallback_used = self._resolve_sample_kernel_names(pc_samples)
kernel_samples = pc_samples.groupby('corr_id', as_index=False)['sample_count'].sum()
kernel_samples = states.groupby('corr_id', as_index=False)['sample_count'].sum()
kernel_samples = kernel_samples.join(
twins.groupby('corr_id')['sample_count'].sum().rename('not_issued_count'),
on='corr_id',
)
kernel_samples['not_issued_count'] = kernel_samples['not_issued_count'].fillna(0)
if corr_to_name:
map_df = pd.DataFrame(
[(k, v) for k, v in corr_to_name.items()],
Expand All @@ -1249,8 +1312,10 @@ def inspect_profile_samples(self, top_n: int = 10):
kernel_table = Table(title=f"PC Sampling Kernels - Top {top_n}")
kernel_table.add_column("Kernel", style="cyan")
kernel_table.add_column("Samples", justify="right")
kernel_table.add_column("Not issued", justify="right")
for _, row in kernel_samples.iterrows():
kernel_table.add_row(str(row['name']), str(int(row['sample_count'])))
kernel_table.add_row(str(row['name']), str(int(row['sample_count'])),
str(int(row['not_issued_count'])))
self.console.print(kernel_table)
if fallback_used > 0:
self.console.print(
Expand Down
Loading
Loading