From cdfd6fea54ee3814a5ddf5a92f7cfddb7b22f7ac Mon Sep 17 00:00:00 2001 From: Yoshifumi Nakamura Date: Fri, 24 Jul 2026 14:40:46 +0900 Subject: [PATCH 1/7] Add GPU NCU plan discovery helpers Signed-off-by: Yoshifumi Nakamura --- .github/workflows/result-server-tests.yml | 5 + docs/guides/add-estimation-package.md | 3 + docs/guides/profiler-level-reference.md | 7 + docs/guides/profiler-support.md | 53 +++ scripts/profiling/generate_ncu_plan.py | 389 ++++++++++++++++++ scripts/profiling/render_ncu_plan_commands.py | 182 ++++++++ .../tests/fixtures/nsys_cuda_gpu_kern_sum.csv | 7 + scripts/tests/test_ncu_plan_generation.sh | 69 ++++ 8 files changed, 715 insertions(+) create mode 100644 scripts/profiling/generate_ncu_plan.py create mode 100644 scripts/profiling/render_ncu_plan_commands.py create mode 100644 scripts/tests/fixtures/nsys_cuda_gpu_kern_sum.csv create mode 100644 scripts/tests/test_ncu_plan_generation.sh diff --git a/.github/workflows/result-server-tests.yml b/.github/workflows/result-server-tests.yml index 10a8687..a1271bb 100644 --- a/.github/workflows/result-server-tests.yml +++ b/.github/workflows/result-server-tests.yml @@ -9,7 +9,9 @@ on: - "scripts/result.sh" - "scripts/result_server/**" - "scripts/estimation/**" + - "scripts/profiling/**" - "scripts/tests/test_bk_profiler.sh" + - "scripts/tests/test_ncu_plan_generation.sh" - "scripts/tests/test_estimation_gpu_kernel_ensemble_average.sh" - "scripts/tests/test_estimation_gpu_kernel_lightgbm_v10.sh" - "scripts/tests/test_estimation_gpu_kernel_mlp_v15.sh" @@ -38,7 +40,9 @@ on: - "scripts/result.sh" - "scripts/result_server/**" - "scripts/estimation/**" + - "scripts/profiling/**" - "scripts/tests/test_bk_profiler.sh" + - "scripts/tests/test_ncu_plan_generation.sh" - "scripts/tests/test_estimation_gpu_kernel_ensemble_average.sh" - "scripts/tests/test_estimation_gpu_kernel_lightgbm_v10.sh" - "scripts/tests/test_estimation_gpu_kernel_mlp_v15.sh" @@ -121,6 +125,7 @@ jobs: - name: Run profiler, profile-data, and estimation shell tests run: | bash scripts/tests/test_bk_profiler.sh + bash scripts/tests/test_ncu_plan_generation.sh bash scripts/tests/test_result_profile_data.sh bash scripts/tests/test_send_results_profile_data.sh bash scripts/tests/test_send_estimate_artifacts.sh diff --git a/docs/guides/add-estimation-package.md b/docs/guides/add-estimation-package.md index 7b99863..2dd5bc1 100644 --- a/docs/guides/add-estimation-package.md +++ b/docs/guides/add-estimation-package.md @@ -123,6 +123,9 @@ section package は prediction CSV の推定実行時間を合算し、その se MLP package は `Execution Time [ns]`、LightGBM package は `O-Execution Time` を主な入力列として扱います。 prepared input CSV から source-side の kernel 実測時間を読める場合は、Estimate JSON の metrics に `source_time_ns`, `predicted_time_ns`, `time_ratio_predicted_over_source`, `speedup_factor_source_over_predicted` を出します。 `ncu` の採取 window はアプリ全体の GPU 区間時間ではなく限定された kernel sample なので、実運用ではこの source/target 比を、profiler overhead のないアプリ区間 timing に掛けて FOM を再構築する想定です。 +GPU kernel を手で事前調査して `kernel_regex` や launch window を固定することは最終形ではありません。 +BenchKit 共通層では `nsys stats` の CUDA kernel summary から `kernel_discovery.json` と `ncu_plan.json` を生成する helper を用意し、推定 package が必要とする NCU 深掘り対象を自動選定する方向に寄せます。 +最初の段階ではアプリ全体の上位 kernel を対象にし、NVTX や app section timing が使える場合は section-aware discovery に拡張します。 CI 配管や app 固有の smoke test は、`programs//estimate.sh` や `scripts/tests/` 側で扱います。 推定 package は、どの app の smoke test で呼ばれるかを知らず、自分が要求する input CSV / prediction CSV / profiler archive だけを見て applicability を判定します。 diff --git a/docs/guides/profiler-level-reference.md b/docs/guides/profiler-level-reference.md index 32add7d..95c621a 100644 --- a/docs/guides/profiler-level-reference.md +++ b/docs/guides/profiler-level-reference.md @@ -54,6 +54,13 @@ Here `both` means text summaries plus CSV reports. Default report behavior for `ncu` is `text`. BenchKit stores the Nsight Compute raw report under `bk_profiler_artifact/raw/rep1/` and, when import succeeds, a text details page under `bk_profiler_artifact/reports/ncu_import_rep1.txt`. +For GPU estimation bring-up, BenchKit also has an offline NCU plan generator: +`scripts/profiling/generate_ncu_plan.py`. +It reads an Nsight Systems CUDA kernel summary CSV and writes `kernel_discovery.json` +plus `ncu_plan.json`. This is intentionally separate from the `single/simple` +level presets: discovery decides which kernels deserve expensive NCU sampling, +while `bk_profiler ncu` still owns the concrete Nsight Compute collection. + ## Portal Summary BenchKit stores profiler metadata in `bk_profiler_artifact/meta.json` inside `padata.tgz`, and also copies a compact summary into `result.json` as `profile_data`. diff --git a/docs/guides/profiler-support.md b/docs/guides/profiler-support.md index 1ef3da5..ed10fc8 100644 --- a/docs/guides/profiler-support.md +++ b/docs/guides/profiler-support.md @@ -95,6 +95,59 @@ binary report も保存したいデバッグ用途では、`BK_PROFILER_ARCHIVE_ MPI launcher 経由の GPU application では、既定で `--target-processes all` を付けて child process も採取対象にする。 追加の kernel filter、section set、NVTX filter などは `BK_PROFILER_ARGS` で `ncu` に渡す。 +### 5.1 GPU kernel discovery と NCU plan + +GPU 性能推定では、アプリ側が kernel 名、launch skip/count、NCU metric list を事前調査して手書きする運用を最終形にしない。 +BenchKit 共通層は、次の段階的 flow を目標にする。 + +1. profiler overhead のない通常実行で FOM と app section timing を取る。 +2. 軽量な discovery 実行で `nsys stats --report cuda_gpu_kern_sum --format csv` 相当の kernel summary を得る。 +3. summary から GPU 時間の上位 kernel を選び、`ncu_plan.json` を生成する。 +4. `ncu_plan.json` に従って、選ばれた代表 kernel launch だけを Nsight Compute で深掘りする。 +5. 推定 package は raw NCU CSV / prepared CSV を入力にし、section time への掛け算に使う source/target ratio を返す。 + +このためのオフライン変換 helper として、`scripts/profiling/generate_ncu_plan.py` を提供する。 +この helper は `nsys` や `ncu` を実行せず、CSV fixture だけでテストできる。 + +```bash +python3 scripts/profiling/generate_ncu_plan.py \ + --nsys-csv results/nsys_cuda_gpu_kern_sum.csv \ + --out-discovery results/kernel_discovery.json \ + --out-plan results/ncu_plan.json \ + --top-k 5 \ + --min-total-time-pct 3 \ + --launch-count 10 +``` + +`kernel_discovery.json` は kernel 名、呼び出し回数、合計 GPU 時間、平均時間を正規化した summary である。 +`ncu_plan.json` は共通 profiler 層や app wrapper が NCU 採取に使える候補 plan で、各 profile に次を含む。 + +- `kernel_name` +- `kernel_match.name_base` +- `kernel_match.pattern` +- `launch_skip` +- `launch_count` +- `metric_set` +- `archive_ncu_report` +- `selection` metadata + +現時点の helper は app section との対応を自動確定しない。 +NVTX range や app-side section timing と接続できる場合は、将来 `section` を埋める拡張で section-aware discovery に進める。 +NVTX がない場合でも、アプリ全体の上位 GPU kernel を自動抽出することで、手書きの kernel regex / skip / count を減らせる。 + +GPU 実機で走らせる前には、plan を `bk_profiler ncu` の dry-run command manifest に変換して確認できる。 + +```bash +python3 scripts/profiling/render_ncu_plan_commands.py \ + --plan results/ncu_plan.json \ + --out results/ncu_commands.json \ + --level detailed \ + -- ./app --input case.inp +``` + +この manifest は profile ごとの `BK_PROFILER_ARGS`、archive path、raw-dir、`bk_profiler ncu` argv を持つ。 +現段階では実行は app wrapper や site runner 側が行い、共通 helper は「自動生成された NCU 採取内容を inspect 可能にする」ことを担当する。 + ## 6. Archive の考え方 `bk_profiler` は archive の中に少なくとも次を置く。 diff --git a/scripts/profiling/generate_ncu_plan.py b/scripts/profiling/generate_ncu_plan.py new file mode 100644 index 0000000..2bf9aab --- /dev/null +++ b/scripts/profiling/generate_ncu_plan.py @@ -0,0 +1,389 @@ +#!/usr/bin/env python3 +"""Generate an Nsight Compute sampling plan from an Nsight Systems summary. + +This helper intentionally does not run ``nsys`` or ``ncu``. It converts a +lightweight CUDA kernel summary, typically exported from +``nsys stats --report cuda_gpu_kern_sum --format csv``, into two JSON files: + +* a normalized kernel-discovery summary +* a compact NCU profile plan for the most important kernels + +The generated plan lets application wrappers avoid hand-maintaining kernel +regular expressions, launch-skip values, and launch counts. +""" + +from __future__ import annotations + +import argparse +import csv +import json +import re +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Iterable + + +SCHEMA_VERSION = 1 + + +@dataclass +class KernelSummary: + name: str + total_time_ns: float + instances: int + avg_time_ns: float | None = None + time_pct: float | None = None + min_time_ns: float | None = None + max_time_ns: float | None = None + stddev_time_ns: float | None = None + + def to_json(self, rank: int, total_gpu_time_ns: float) -> dict[str, Any]: + pct = self.time_pct + if pct is None and total_gpu_time_ns > 0: + pct = self.total_time_ns / total_gpu_time_ns * 100.0 + return { + "rank": rank, + "name": self.name, + "total_time_ns": self.total_time_ns, + "time_pct": pct, + "instances": self.instances, + "avg_time_ns": self.avg_time_ns, + "min_time_ns": self.min_time_ns, + "max_time_ns": self.max_time_ns, + "stddev_time_ns": self.stddev_time_ns, + } + + +def _normalize_header(value: str) -> str: + return re.sub(r"[^a-z0-9]+", "_", value.strip().lower()).strip("_") + + +def _parse_number(value: Any) -> float | None: + if value is None: + return None + text = str(value).strip() + if not text or text in {"-", "N/A", "n/a", "nan", "NaN"}: + return None + text = text.replace(",", "") + try: + return float(text) + except ValueError: + return None + + +def _parse_int(value: Any) -> int: + parsed = _parse_number(value) + if parsed is None: + return 0 + return int(parsed) + + +def _time_unit_multiplier_ns(header: str) -> float: + normalized = _normalize_header(header) + if normalized.endswith("_ns") or normalized.endswith("_nsec"): + return 1.0 + if normalized.endswith("_us") or normalized.endswith("_usec"): + return 1_000.0 + if normalized.endswith("_ms") or normalized.endswith("_msec"): + return 1_000_000.0 + if normalized.endswith("_s") or normalized.endswith("_sec"): + return 1_000_000_000.0 + # Nsight Systems usually writes "Total Time (ns)" or "Avg (ns)"; if the + # unit is omitted in a fixture, keep the value as nanoseconds. + return 1.0 + + +def _pick(row: dict[str, str], aliases: Iterable[str]) -> tuple[str, str] | None: + normalized = {_normalize_header(key): key for key in row} + for alias in aliases: + key = normalized.get(alias) + if key is not None: + return key, row[key] + return None + + +def parse_nsys_kernel_csv(csv_path: Path) -> list[KernelSummary]: + """Parse an Nsight Systems CUDA kernel summary CSV. + + The parser accepts the common ``cuda_gpu_kern_sum`` spelling and a small + set of alias headers so that checked-in fixtures remain readable. + """ + + with csv_path.open(newline="", encoding="utf-8-sig") as handle: + lines = [line for line in handle if line.strip()] + + header_index = 0 + for index, line in enumerate(lines): + try: + candidate = next(csv.reader([line])) + except csv.Error: + continue + normalized = {_normalize_header(item) for item in candidate} + has_name = bool(normalized & {"name", "kernel_name", "demangled_name"}) + has_total = bool( + normalized + & { + "total_time_ns", + "total_time_nsec", + "total_time", + "time_ns", + "total_ns", + } + ) + if has_name and has_total: + header_index = index + break + + rows = list(csv.DictReader(lines[header_index:])) + + kernels: list[KernelSummary] = [] + for row in rows: + name_item = _pick(row, ["name", "kernel_name", "demangled_name"]) + total_item = _pick( + row, + [ + "total_time_ns", + "total_time_nsec", + "total_time", + "time_ns", + "total_ns", + ], + ) + if name_item is None or total_item is None: + continue + + total_value = _parse_number(total_item[1]) + if total_value is None: + continue + total_time_ns = total_value * _time_unit_multiplier_ns(total_item[0]) + + instances_item = _pick(row, ["instances", "calls", "count"]) + avg_item = _pick(row, ["avg_ns", "avg_nsec", "avg_time_ns", "avg", "average_ns"]) + pct_item = _pick(row, ["time", "time_pct", "total_time_pct", "percent", "time_percent"]) + min_item = _pick(row, ["min_ns", "min_nsec", "min"]) + max_item = _pick(row, ["max_ns", "max_nsec", "max"]) + std_item = _pick(row, ["stddev_ns", "std_dev_ns", "stddev", "stdev_ns"]) + + def time_or_none(item: tuple[str, str] | None) -> float | None: + if item is None: + return None + parsed = _parse_number(item[1]) + if parsed is None: + return None + return parsed * _time_unit_multiplier_ns(item[0]) + + kernels.append( + KernelSummary( + name=name_item[1].strip(), + total_time_ns=total_time_ns, + instances=_parse_int(instances_item[1]) if instances_item else 0, + avg_time_ns=time_or_none(avg_item), + time_pct=_parse_number(pct_item[1]) if pct_item else None, + min_time_ns=time_or_none(min_item), + max_time_ns=time_or_none(max_item), + stddev_time_ns=time_or_none(std_item), + ) + ) + + kernels.sort(key=lambda item: item.total_time_ns, reverse=True) + return kernels + + +def _regex_exact_kernel_name(kernel_name: str) -> str: + return f"regex:^{re.escape(kernel_name)}$" + + +def _slugify_kernel_name(kernel_name: str, index: int) -> str: + slug = re.sub(r"[^A-Za-z0-9]+", "_", kernel_name).strip("_").lower() + slug = re.sub(r"_+", "_", slug) + if not slug: + slug = "kernel" + return f"k{index:03d}_{slug[:48]}" + + +def _launch_skip(instances: int, warmup_fraction: float, max_skip: int) -> int: + if instances <= 1: + return 0 + skip = int(instances * warmup_fraction) + skip = min(skip, max_skip) + return min(skip, instances - 1) + + +def _launch_count(instances: int, skip: int, requested: int) -> int: + remaining = max(instances - skip, 1) + return max(1, min(requested, remaining)) + + +def select_kernels( + kernels: list[KernelSummary], + *, + top_k: int, + min_total_time_pct: float, + min_instances: int, +) -> list[KernelSummary]: + total = sum(item.total_time_ns for item in kernels) + selected: list[KernelSummary] = [] + for item in kernels: + pct = item.time_pct + if pct is None and total > 0: + pct = item.total_time_ns / total * 100.0 + if item.instances < min_instances: + continue + if pct is not None and pct < min_total_time_pct: + continue + selected.append(item) + if len(selected) >= top_k: + break + return selected + + +def build_discovery_json(csv_path: Path, kernels: list[KernelSummary]) -> dict[str, Any]: + total_gpu_time_ns = sum(item.total_time_ns for item in kernels) + return { + "schema_version": SCHEMA_VERSION, + "source": { + "tool": "nsys", + "report": "cuda_gpu_kern_sum", + "path": str(csv_path), + }, + "summary": { + "kernel_count": len(kernels), + "total_gpu_time_ns": total_gpu_time_ns, + }, + "kernels": [ + item.to_json(rank=index, total_gpu_time_ns=total_gpu_time_ns) + for index, item in enumerate(kernels, start=1) + ], + } + + +def build_ncu_plan_json( + *, + discovery_path: Path | None, + selected: list[KernelSummary], + all_kernels: list[KernelSummary], + metric_set: str, + launch_count: int, + warmup_fraction: float, + max_launch_skip: int, + archive_ncu_report: bool, +) -> dict[str, Any]: + total_gpu_time_ns = sum(item.total_time_ns for item in all_kernels) + profiles = [] + for index, item in enumerate(selected, start=1): + skip = _launch_skip(item.instances, warmup_fraction, max_launch_skip) + count = _launch_count(item.instances, skip, launch_count) + pct = item.time_pct + if pct is None and total_gpu_time_ns > 0: + pct = item.total_time_ns / total_gpu_time_ns * 100.0 + profiles.append( + { + "name": _slugify_kernel_name(item.name, index), + "section": None, + "kernel_name": item.name, + "kernel_match": { + "name_base": "demangled", + "mode": "regex", + "pattern": _regex_exact_kernel_name(item.name), + }, + "launch_skip": skip, + "launch_count": count, + "metric_set": metric_set, + "archive_ncu_report": archive_ncu_report, + "selection": { + "rank": index, + "total_time_ns": item.total_time_ns, + "time_pct": pct, + "instances": item.instances, + "avg_time_ns": item.avg_time_ns, + }, + } + ) + return { + "schema_version": SCHEMA_VERSION, + "source": { + "discovery_path": str(discovery_path) if discovery_path else None, + "tool": "nsys", + "report": "cuda_gpu_kern_sum", + }, + "policy": { + "metric_set": metric_set, + "requested_launch_count": launch_count, + "warmup_fraction": warmup_fraction, + "max_launch_skip": max_launch_skip, + "archive_ncu_report": archive_ncu_report, + }, + "profiles": profiles, + } + + +def write_json(path: Path, payload: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--nsys-csv", required=True, type=Path, help="nsys cuda_gpu_kern_sum CSV") + parser.add_argument("--out-discovery", type=Path, help="output kernel discovery JSON") + parser.add_argument("--out-plan", type=Path, help="output NCU plan JSON") + parser.add_argument("--top-k", type=int, default=5, help="maximum selected kernels") + parser.add_argument("--min-total-time-pct", type=float, default=3.0, help="minimum GPU time percentage") + parser.add_argument("--min-instances", type=int, default=1, help="minimum launch count") + parser.add_argument("--launch-count", type=int, default=10, help="requested NCU launches per kernel") + parser.add_argument("--warmup-fraction", type=float, default=0.10, help="fraction of launches to skip") + parser.add_argument("--max-launch-skip", type=int, default=100, help="maximum generated launch skip") + parser.add_argument("--metric-set", default="gpu_kernel_estimation", help="NCU metric-set label") + parser.add_argument("--archive-ncu-report", action="store_true", help="request .ncu-rep archive retention") + parser.add_argument("--print-plan", action="store_true", help="print generated plan to stdout") + args = parser.parse_args(argv) + + if args.top_k < 1: + parser.error("--top-k must be >= 1") + if args.launch_count < 1: + parser.error("--launch-count must be >= 1") + if args.warmup_fraction < 0: + parser.error("--warmup-fraction must be >= 0") + if args.max_launch_skip < 0: + parser.error("--max-launch-skip must be >= 0") + + kernels = parse_nsys_kernel_csv(args.nsys_csv) + if not kernels: + print(f"No CUDA kernels parsed from {args.nsys_csv}", file=sys.stderr) + return 1 + + selected = select_kernels( + kernels, + top_k=args.top_k, + min_total_time_pct=args.min_total_time_pct, + min_instances=args.min_instances, + ) + if not selected: + print("No CUDA kernels matched the selection policy", file=sys.stderr) + return 1 + + discovery = build_discovery_json(args.nsys_csv, kernels) + plan = build_ncu_plan_json( + discovery_path=args.out_discovery, + selected=selected, + all_kernels=kernels, + metric_set=args.metric_set, + launch_count=args.launch_count, + warmup_fraction=args.warmup_fraction, + max_launch_skip=args.max_launch_skip, + archive_ncu_report=args.archive_ncu_report, + ) + + if args.out_discovery: + write_json(args.out_discovery, discovery) + if args.out_plan: + write_json(args.out_plan, plan) + if args.print_plan or not args.out_plan: + print(json.dumps(plan, indent=2, ensure_ascii=False)) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/profiling/render_ncu_plan_commands.py b/scripts/profiling/render_ncu_plan_commands.py new file mode 100644 index 0000000..06ee377 --- /dev/null +++ b/scripts/profiling/render_ncu_plan_commands.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +"""Render a BenchKit NCU plan into concrete ``bk_profiler ncu`` commands. + +This helper is deliberately dry-run friendly. It does not run Nsight Compute; +it translates ``ncu_plan.json`` profiles into the environment variables and +``bk_profiler`` command lines that a site/app wrapper can execute on a GPU +node. Keeping this step inspectable makes the automatic discovery flow safer +before we connect it to real runners. +""" + +from __future__ import annotations + +import argparse +import json +import shlex +import sys +from pathlib import Path +from typing import Any + + +def _shell_join(args: list[str]) -> str: + return " ".join(shlex.quote(str(item)) for item in args) + + +def _truth(value: Any) -> str: + return "true" if bool(value) else "false" + + +def _profile_args(profile: dict[str, Any]) -> list[str]: + match = profile.get("kernel_match") if isinstance(profile.get("kernel_match"), dict) else {} + name_base = str(match.get("name_base") or "demangled") + pattern = str(match.get("pattern") or "") + if not pattern: + raise ValueError(f"profile {profile.get('name')!r} is missing kernel_match.pattern") + + args = [ + "--kernel-name-base", + name_base, + "--kernel-name", + pattern, + "--launch-skip", + str(int(profile.get("launch_skip") or 0)), + "--launch-count", + str(int(profile.get("launch_count") or 1)), + ] + return args + + +def _command_record( + profile: dict[str, Any], + *, + level: str, + archive_dir: Path, + raw_dir_prefix: str, + command: list[str], +) -> dict[str, Any]: + name = str(profile.get("name") or "profile") + archive = archive_dir / f"padata_{name}.tgz" + raw_dir = f"{raw_dir_prefix}_{name}" + profiler_args = _profile_args(profile) + archive_report = bool(profile.get("archive_ncu_report", False)) + + env = { + "BK_PROFILER_ARGS": _shell_join(profiler_args), + "BK_PROFILER_NCU_RAW_CSV": "true", + "BK_PROFILER_ARCHIVE_NCU_REPORT": _truth(archive_report), + } + argv = [ + "bk_profiler", + "ncu", + "--level", + level, + "--archive", + str(archive), + "--raw-dir", + raw_dir, + "--", + *command, + ] + return { + "name": name, + "kernel_name": profile.get("kernel_name"), + "section": profile.get("section"), + "metric_set": profile.get("metric_set"), + "archive": str(archive), + "raw_dir": raw_dir, + "env": env, + "bk_profiler_args": profiler_args, + "argv": argv, + "selection": profile.get("selection") if isinstance(profile.get("selection"), dict) else {}, + } + + +def build_command_manifest( + plan: dict[str, Any], + *, + level: str, + archive_dir: Path, + raw_dir_prefix: str, + command: list[str], +) -> dict[str, Any]: + profiles = plan.get("profiles") + if not isinstance(profiles, list) or not profiles: + raise ValueError("ncu plan has no profiles") + + commands = [ + _command_record( + profile, + level=level, + archive_dir=archive_dir, + raw_dir_prefix=raw_dir_prefix, + command=command, + ) + for profile in profiles + if isinstance(profile, dict) + ] + return { + "schema_version": 1, + "source": { + "plan_schema_version": plan.get("schema_version"), + "plan_source": plan.get("source"), + }, + "execution": { + "tool": "bk_profiler", + "profiler": "ncu", + "level": level, + "command": command, + }, + "commands": commands, + } + + +def print_shell(manifest: dict[str, Any]) -> None: + for record in manifest["commands"]: + for key, value in record["env"].items(): + print(f"export {key}={shlex.quote(str(value))}") + print(_shell_join(record["argv"])) + print() + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--plan", required=True, type=Path, help="input ncu_plan.json") + parser.add_argument("--out", type=Path, help="write command manifest JSON") + parser.add_argument("--format", choices=["json", "shell"], default="shell", help="stdout format") + parser.add_argument("--level", default="detailed", help="bk_profiler ncu level") + parser.add_argument("--archive-dir", type=Path, default=Path("results"), help="archive directory") + parser.add_argument("--raw-dir-prefix", default="ncu_plan", help="raw-dir prefix") + parser.add_argument("command", nargs=argparse.REMAINDER, help="profiled command after --") + args = parser.parse_args(argv) + + command = args.command + if command and command[0] == "--": + command = command[1:] + if not command: + parser.error("profiled command is required after --") + + with args.plan.open(encoding="utf-8") as handle: + plan = json.load(handle) + + manifest = build_command_manifest( + plan, + level=args.level, + archive_dir=args.archive_dir, + raw_dir_prefix=args.raw_dir_prefix, + command=command, + ) + + if args.out: + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(manifest, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + + if args.format == "json": + print(json.dumps(manifest, indent=2, ensure_ascii=False)) + else: + print_shell(manifest) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/tests/fixtures/nsys_cuda_gpu_kern_sum.csv b/scripts/tests/fixtures/nsys_cuda_gpu_kern_sum.csv new file mode 100644 index 0000000..67e7262 --- /dev/null +++ b/scripts/tests/fixtures/nsys_cuda_gpu_kern_sum.csv @@ -0,0 +1,7 @@ +CUDA Kernel Summary +"Time (%)","Total Time (ns)","Instances","Avg (ns)","Med (ns)","Min (ns)","Max (ns)","StdDev (ns)","Name" +63.5,"6,350,000",120,52916.67,52000,48000,65000,2400,"void kern_compute_force_nonbond_table_linear_univ__inter_cell(float*)" +24.0,"2,400,000",120,20000,19800,18000,25000,1300,"void kern_compute_force_nonbond_table_linear_univ__intra_cell(float*)" +8.0,"800,000",12,66666.67,65000,60000,90000,6000,"void kern_build_pairlist<4, 256>(float*)" +2.0,"200,000",40,5000,4900,4500,6000,300,"void tiny_kernel()" +0.5,"50,000",1,50000,50000,50000,50000,0,"void one_off_setup_kernel()" diff --git a/scripts/tests/test_ncu_plan_generation.sh b/scripts/tests/test_ncu_plan_generation.sh new file mode 100644 index 0000000..d96fdcb --- /dev/null +++ b/scripts/tests/test_ncu_plan_generation.sh @@ -0,0 +1,69 @@ +#!/bin/bash +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +REPO_DIR=$(cd "${SCRIPT_DIR}/../.." && pwd) + +PYTHON_BIN="${PYTHON_BIN:-python3}" +TMP_DIR=$(mktemp -d) +trap 'rm -rf "${TMP_DIR}"' EXIT + +"${PYTHON_BIN}" "${REPO_DIR}/scripts/profiling/generate_ncu_plan.py" \ + --nsys-csv "${REPO_DIR}/scripts/tests/fixtures/nsys_cuda_gpu_kern_sum.csv" \ + --out-discovery "${TMP_DIR}/kernel_discovery.json" \ + --out-plan "${TMP_DIR}/ncu_plan.json" \ + --top-k 3 \ + --min-total-time-pct 3 \ + --launch-count 10 \ + --warmup-fraction 0.10 \ + --max-launch-skip 20 + +"${PYTHON_BIN}" "${REPO_DIR}/scripts/profiling/render_ncu_plan_commands.py" \ + --plan "${TMP_DIR}/ncu_plan.json" \ + --out "${TMP_DIR}/ncu_commands.json" \ + --format json \ + --level detailed \ + --archive-dir results \ + --raw-dir-prefix ncu_auto \ + -- ./app --input case.inp >/dev/null + +test -f "${TMP_DIR}/kernel_discovery.json" +test -f "${TMP_DIR}/ncu_plan.json" +test -f "${TMP_DIR}/ncu_commands.json" + +jq -e ' + .schema_version == 1 and + .summary.kernel_count == 5 and + .kernels[0].instances == 120 and + (.kernels[0].name | contains("inter_cell")) +' "${TMP_DIR}/kernel_discovery.json" >/dev/null + +jq -e ' + .schema_version == 1 and + .policy.metric_set == "gpu_kernel_estimation" and + (.profiles | length) == 3 and + .profiles[0].launch_skip == 12 and + .profiles[0].launch_count == 10 and + .profiles[0].kernel_match.name_base == "demangled" and + (.profiles[0].kernel_match.pattern | startswith("regex:^")) and + (.profiles[2].kernel_name | contains("build_pairlist")) +' "${TMP_DIR}/ncu_plan.json" >/dev/null + +if jq -e '.profiles[].kernel_name | contains("tiny_kernel")' "${TMP_DIR}/ncu_plan.json" >/dev/null; then + echo "tiny low-impact kernel should not be selected" >&2 + exit 1 +fi + +jq -e ' + .schema_version == 1 and + .execution.profiler == "ncu" and + .execution.level == "detailed" and + (.commands | length) == 3 and + .commands[0].env.BK_PROFILER_NCU_RAW_CSV == "true" and + (.commands[0].env.BK_PROFILER_ARGS | contains("--kernel-name-base")) and + (.commands[0].argv | index("bk_profiler") == 0) and + (.commands[0].argv | index("--archive") != null) and + (.commands[0].argv | index("./app") != null) +' "${TMP_DIR}/ncu_commands.json" >/dev/null + +echo "ncu plan generation tests passed" From a7fa71c60187ef2a44443899ee1b2f35579070ca Mon Sep 17 00:00:00 2001 From: Yoshifumi Nakamura Date: Fri, 24 Jul 2026 15:45:53 +0900 Subject: [PATCH 2/7] Connect GENESIS to automatic NCU plans Signed-off-by: Yoshifumi Nakamura --- docs/guides/profiler-support.md | 9 +- programs/genesis/README.md | 22 +++ programs/genesis/run.sh | 195 +++++++++++++++++++- scripts/profiling/generate_ncu_plan.py | 64 +++++-- scripts/profiling/iter_ncu_plan_profiles.py | 66 +++++++ scripts/tests/test_ncu_plan_generation.sh | 57 +++++- 6 files changed, 391 insertions(+), 22 deletions(-) create mode 100644 scripts/profiling/iter_ncu_plan_profiles.py diff --git a/docs/guides/profiler-support.md b/docs/guides/profiler-support.md index ed10fc8..a0e42c3 100644 --- a/docs/guides/profiler-support.md +++ b/docs/guides/profiler-support.md @@ -114,13 +114,16 @@ python3 scripts/profiling/generate_ncu_plan.py \ --nsys-csv results/nsys_cuda_gpu_kern_sum.csv \ --out-discovery results/kernel_discovery.json \ --out-plan results/ncu_plan.json \ - --top-k 5 \ - --min-total-time-pct 3 \ + --top-k 0 \ --launch-count 10 ``` `kernel_discovery.json` は kernel 名、呼び出し回数、合計 GPU 時間、平均時間を正規化した summary である。 `ncu_plan.json` は共通 profiler 層や app wrapper が NCU 採取に使える候補 plan で、各 profile に次を含む。 +`--top-k 0` は discovery 調査用に全 kernel を plan へ残す指定である。 +NCU 実行時は、NSYS で同定した kernel に対して `launch_skip=1` / `launch_count=10` +程度の小さな window から始める。`discovery_gpu_time_pct` は NSYS で観測された +GPU kernel 時間内の割合であり、app FOM 全体に対する割合ではない。 - `kernel_name` - `kernel_match.name_base` @@ -128,6 +131,8 @@ python3 scripts/profiling/generate_ncu_plan.py \ - `launch_skip` - `launch_count` - `metric_set` +- `selection.source_gpu_duration_ns` +- `selection.discovery_gpu_time_pct` - `archive_ncu_report` - `selection` metadata diff --git a/programs/genesis/README.md b/programs/genesis/README.md index 42d9c43..47e2bf9 100644 --- a/programs/genesis/README.md +++ b/programs/genesis/README.md @@ -63,6 +63,28 @@ BK_GENESIS_NCU__NSTEPS Legacy single-window collection can be requested with `BK_GENESIS_NCU_KERNEL_REGEX`; the wrapper treats it as a `custom` profile. +The default automatic mode derives candidate NCU windows from an Nsight Systems +CUDA kernel summary and then executes the generated NCU plan: + +```bash +BK_GENESIS_NCU_PROFILE_MODE=discovery +``` + +In this mode, `run.sh` keeps the normal unprofiled benchmark run, then runs a +short NSYS discovery pass, writes `results/kernel_discovery.json` and +`results/ncu_plan.json`, and runs the selected NCU windows. The generated NCU +profiles default to the top three GPU-time kernels with `launch_skip=1` and +`launch_count=10`; the NCU archives are registered as section artifacts and are +used by the GPU estimation packages to compute source/target kernel time ratios. + +Use `BK_GENESIS_NCU_PROFILE_MODE=discovery-only` when investigating NSYS output +without paying the NCU cost. That mode writes the full discovery summary and +plan but skips NCU collection. If the NSYS CSV has already been created by +site-local tooling, set +`BK_GENESIS_NCU_DISCOVERY_CSV=/path/to/cuda_gpu_kern_sum.csv` to skip the NSYS +pass and only generate the NCU plan. Manual mode still uses the configured +`inter intra pairlist` windows above. + ## Estimation Sections GENESIS treats the log `dynamics` time as the FOM. The app-side parser diff --git a/programs/genesis/run.sh b/programs/genesis/run.sh index 6389d0f..bb95ae9 100644 --- a/programs/genesis/run.sh +++ b/programs/genesis/run.sh @@ -274,7 +274,8 @@ genesis_run_ncu_profile() { local launch_skip="$4" local launch_count="$5" local profiler_level="$6" - shift 6 + local section_name="$7" + shift 7 local archive_path="${resultsdir}/padata_${profile_slug}.tgz" local archive_rel_path="results/padata_${profile_slug}.tgz" @@ -282,7 +283,6 @@ genesis_run_ncu_profile() { local profile_log="${resultsdir}/log_${header}_ncu_${profile_slug}.txt" local ncu_args local profile_status - local section_name local old_profiler_args="${BK_PROFILER_ARGS:-}" local old_profiler_raw_csv="${BK_PROFILER_NCU_RAW_CSV:-}" local had_profiler_args=0 @@ -305,7 +305,7 @@ genesis_run_ncu_profile() { --level "$profiler_level" \ --archive "$archive_path" \ --raw-dir "$raw_dir" \ - -- "$@" 2>&1 | tee "$profile_log" + -- "$@" &1 | tee "$profile_log" profile_status=${PIPESTATUS[0]} set -e @@ -325,8 +325,9 @@ genesis_run_ncu_profile() { return "$profile_status" fi - section_name=$(genesis_profile_section_name "$profile_name") - genesis_register_section_artifact "$section_name" "$archive_rel_path" + if [ -n "$section_name" ]; then + genesis_register_section_artifact "$section_name" "$archive_rel_path" + fi } genesis_run_ncu_profiles() { @@ -378,10 +379,173 @@ genesis_run_ncu_profiles() { profile_input=$(genesis_prepare_ncu_input "${profile_cmd[$last_index]}" "$profile_name" "$profile_slug" "$profile_key") profile_cmd[$last_index]="$profile_input" - genesis_run_ncu_profile "$profile_name" "$profile_slug" "$kernel_regex" "$launch_skip" "$launch_count" "$profiler_level" "${profile_cmd[@]}" + genesis_run_ncu_profile "$profile_name" "$profile_slug" "$kernel_regex" "$launch_skip" "$launch_count" "$profiler_level" "$(genesis_profile_section_name "$profile_name")" "${profile_cmd[@]}" done } +genesis_python_bin() { + if [ -n "${PYTHON_BIN:-}" ]; then + printf '%s\n' "$PYTHON_BIN" + else + printf '%s\n' "python3" + fi +} + +genesis_ncu_profile_mode() { + printf '%s\n' "${BK_GENESIS_NCU_PROFILE_MODE:-discovery}" +} + +genesis_generate_ncu_plan() { + local profiler_level="$1" + shift + local python_bin + local discovery_csv="${BK_GENESIS_NCU_DISCOVERY_CSV:-${BK_GENESIS_NSYS_KERNEL_SUMMARY_CSV:-}}" + local discovery_json="${resultsdir}/kernel_discovery.json" + local plan_json="${resultsdir}/ncu_plan.json" + local nsys_base="${resultsdir}/nsys_kernel_discovery" + local nsys_report="${nsys_base}.nsys-rep" + local nsys_csv="${resultsdir}/nsys_cuda_gpu_kern_sum.csv" + local nsys_log="${resultsdir}/log_${header}_nsys_discovery.txt" + local discovery_cmd + local last_index + local discovery_input + local generated_csv + local nsys_status + local plan_top_k="${BK_GENESIS_NCU_PLAN_TOP_K:-}" + + python_bin=$(genesis_python_bin) + if ! command -v "$python_bin" >/dev/null 2>&1; then + echo "GENESIS NCU discovery requires ${python_bin} for plan generation." >&2 + return 1 + fi + + if [ -z "$discovery_csv" ]; then + if ! command -v nsys >/dev/null 2>&1; then + echo "GENESIS NCU discovery requires nsys, or set BK_GENESIS_NCU_DISCOVERY_CSV to an existing cuda_gpu_kern_sum CSV." >&2 + return 1 + fi + + discovery_cmd=("$@") + last_index=$((${#discovery_cmd[@]} - 1)) + if [ "$last_index" -lt 0 ]; then + echo "GENESIS NCU discovery has no command to run." >&2 + return 1 + fi + discovery_input=$(genesis_prepare_ncu_input "${discovery_cmd[$last_index]}" "discovery" "discovery" "DISCOVERY") + discovery_cmd[$last_index]="$discovery_input" + + echo "Running GENESIS NSYS kernel discovery for automatic NCU plan generation level=${profiler_level}" >&2 + set +e + nsys profile \ + --force-overwrite=true \ + --trace=cuda \ + --sample=none \ + -o "$nsys_base" \ + "${discovery_cmd[@]}" 2>&1 | tee "$nsys_log" >&2 + nsys_status=${PIPESTATUS[0]} + set -e + if [ "$nsys_status" -ne 0 ]; then + echo "GENESIS NSYS kernel discovery failed with status ${nsys_status}" >&2 + return "$nsys_status" + fi + + if [ ! -f "$nsys_report" ]; then + echo "GENESIS NSYS report was not created: ${nsys_report}" >&2 + return 1 + fi + nsys stats --report cuda_gpu_kern_sum --format csv --output "$nsys_csv" "$nsys_report" >/dev/null + generated_csv=$(find "${resultsdir}" -maxdepth 1 -type f \( -name 'nsys_cuda_gpu_kern_sum*.csv' -o -name 'nsys_cuda_gpu_kern_sum*.csv.*' \) | sort | head -n 1) + discovery_csv="${generated_csv:-$nsys_csv}" + echo "GENESIS NSYS CUDA kernel summary CSV: ${discovery_csv}" >&2 + echo "---- GENESIS NSYS CUDA kernel summary begin ----" >&2 + cat "$discovery_csv" >&2 + echo "---- GENESIS NSYS CUDA kernel summary end ----" >&2 + fi + + if [ ! -f "$discovery_csv" ]; then + echo "GENESIS NCU discovery CSV does not exist: ${discovery_csv}" >&2 + return 1 + fi + if [ -z "$plan_top_k" ]; then + case "$(genesis_ncu_profile_mode)" in + discovery-only|auto-discovery-only) + plan_top_k=0 + ;; + *) + plan_top_k=3 + ;; + esac + fi + + "$python_bin" "${SCRIPT_DIR}/scripts/profiling/generate_ncu_plan.py" \ + --nsys-csv "$discovery_csv" \ + --out-discovery "$discovery_json" \ + --out-plan "$plan_json" \ + --top-k "$plan_top_k" \ + --min-total-time-pct "${BK_GENESIS_NCU_PLAN_MIN_TOTAL_TIME_PCT:-0}" \ + --min-instances "${BK_GENESIS_NCU_PLAN_MIN_INSTANCES:-1}" \ + --launch-count "${BK_GENESIS_NCU_PLAN_LAUNCH_COUNT:-${BK_GENESIS_NCU_LAUNCH_COUNT:-10}}" \ + --warmup-fraction "${BK_GENESIS_NCU_PLAN_WARMUP_FRACTION:-0}" \ + --max-launch-skip "${BK_GENESIS_NCU_PLAN_MAX_LAUNCH_SKIP:-1}" \ + --metric-set "${BK_GENESIS_NCU_PLAN_METRIC_SET:-gpu_kernel_estimation}" + + echo "GENESIS kernel discovery JSON: ${discovery_json}" >&2 + echo "---- GENESIS kernel discovery JSON begin ----" >&2 + cat "$discovery_json" >&2 + echo "---- GENESIS kernel discovery JSON end ----" >&2 + echo "GENESIS NCU plan JSON: ${plan_json}" >&2 + echo "---- GENESIS NCU plan JSON begin ----" >&2 + cat "$plan_json" >&2 + echo "---- GENESIS NCU plan JSON end ----" >&2 + + printf '%s\n' "$plan_json" +} + +genesis_run_ncu_plan_profiles() { + local plan_json="$1" + local profiler_level="$2" + shift 2 + local python_bin + local profile_name + local section_name + local kernel_regex + local launch_skip + local launch_count + local profile_slug + local profile_key + local profile_cmd + local last_index + local profile_input + local profile_seen=0 + local profile_rows=() + local profile_row + + python_bin=$(genesis_python_bin) + mapfile -t profile_rows < <("$python_bin" "${SCRIPT_DIR}/scripts/profiling/iter_ncu_plan_profiles.py" --plan "$plan_json") + for profile_row in "${profile_rows[@]}"; do + IFS=$'\t' read -r profile_name section_name kernel_regex launch_skip launch_count <<< "$profile_row" + if [ -z "$profile_name" ] || [ -z "$kernel_regex" ]; then + continue + fi + profile_seen=$((profile_seen + 1)) + profile_slug=$(genesis_profile_slug "$profile_name") + profile_key=$(genesis_profile_key "$profile_name") + profile_cmd=("$@") + last_index=$((${#profile_cmd[@]} - 1)) + if [ "$last_index" -lt 0 ]; then + echo "GENESIS NCU plan profile '${profile_name}' has no command to run." >&2 + return 1 + fi + profile_input=$(genesis_prepare_ncu_input "${profile_cmd[$last_index]}" "$profile_name" "$profile_slug" "$profile_key") + profile_cmd[$last_index]="$profile_input" + genesis_run_ncu_profile "$profile_name" "$profile_slug" "$kernel_regex" "$launch_skip" "$launch_count" "$profiler_level" "$section_name" "${profile_cmd[@]}" + done + if [ "$profile_seen" -eq 0 ]; then + echo "GENESIS NCU plan has no executable profiles: ${plan_json}" >&2 + return 1 + fi +} + # Shared GH200-class run path. The env_prefix pattern mirrors build.sh so each # site can override modules, MPI launcher, GPU visibility, and profiler policy # independently while keeping the benchmark invocation identical. @@ -460,7 +624,24 @@ run_genesis_gh200_gpu() { return 1 fi echo "Running ${system_name} additional NCU acquisition profiles level=${genesis_profiler_level}" - genesis_run_ncu_profiles "$genesis_profiler_level" "${mpi_cmd[@]}" ./${binary} ${input}.sub + case "$(genesis_ncu_profile_mode)" in + manual|configured) + genesis_run_ncu_profiles "$genesis_profiler_level" "${mpi_cmd[@]}" ./${binary} ${input}.sub + ;; + discovery|auto) + genesis_plan_json=$(genesis_generate_ncu_plan "$genesis_profiler_level" "${mpi_cmd[@]}" ./${binary} ${input}.sub) + genesis_run_ncu_plan_profiles "$genesis_plan_json" "$genesis_profiler_level" "${mpi_cmd[@]}" ./${binary} ${input}.sub + ;; + discovery-only|auto-discovery-only) + genesis_generate_ncu_plan "$genesis_profiler_level" "${mpi_cmd[@]}" ./${binary} ${input}.sub >/dev/null + echo "Genesis ${system_name}: completed NSYS kernel discovery; skipping NCU profile execution." + ;; + *) + echo "Genesis ${system_name}: unsupported BK_GENESIS_NCU_PROFILE_MODE='$(genesis_ncu_profile_mode)'." >&2 + echo "Use manual, discovery, or discovery-only." >&2 + return 1 + ;; + esac fi } diff --git a/scripts/profiling/generate_ncu_plan.py b/scripts/profiling/generate_ncu_plan.py index 2bf9aab..f94fae4 100644 --- a/scripts/profiling/generate_ncu_plan.py +++ b/scripts/profiling/generate_ncu_plan.py @@ -46,8 +46,11 @@ def to_json(self, rank: int, total_gpu_time_ns: float) -> dict[str, Any]: "rank": rank, "name": self.name, "total_time_ns": self.total_time_ns, + "source_gpu_duration_ns": self.total_time_ns, "time_pct": pct, + "discovery_gpu_time_pct": pct, "instances": self.instances, + "source_launches": self.instances, "avg_time_ns": self.avg_time_ns, "min_time_ns": self.min_time_ns, "max_time_ns": self.max_time_ns, @@ -190,8 +193,41 @@ def time_or_none(item: tuple[str, str] | None) -> float | None: return kernels -def _regex_exact_kernel_name(kernel_name: str) -> str: - return f"regex:^{re.escape(kernel_name)}$" +def _kernel_match_token(kernel_name: str) -> str: + """Return a shell-safe demangled-name substring for NCU kernel matching. + + Nsight Systems summaries often include the full demangled signature. Passing + that signature through shell variables is fragile because argument lists and + template parameters contain spaces. NCU only needs a stable substring, so + prefer GENESIS-style kernel family names and fall back to the function stem. + """ + + if "build_pairlist" in kernel_name: + return "build_pairlist" + if "energyforce_inter_cell" in kernel_name: + return "energyforce_inter_cell" + if "energyforce_intra_cell" in kernel_name: + return "energyforce_intra_cell" + if "force_inter_cell" in kernel_name: + return "force_inter_cell" + if "force_intra_cell" in kernel_name: + return "force_intra_cell" + if "inter_cell" in kernel_name: + return "inter_cell" + if "intra_cell" in kernel_name: + return "intra_cell" + + stem = kernel_name.split("(", 1)[0].strip() + stem = re.sub(r"<.*>$", "", stem).strip() + parts = stem.split() + if parts: + stem = parts[-1] + return stem or kernel_name.strip() + + +def _regex_kernel_name(kernel_name: str) -> str: + token = _kernel_match_token(kernel_name) + return f"regex:.*{re.escape(token)}.*" def _slugify_kernel_name(kernel_name: str, index: int) -> str: @@ -205,7 +241,9 @@ def _slugify_kernel_name(kernel_name: str, index: int) -> str: def _launch_skip(instances: int, warmup_fraction: float, max_skip: int) -> int: if instances <= 1: return 0 - skip = int(instances * warmup_fraction) + if max_skip <= 0: + return 0 + skip = max(1, int(instances * warmup_fraction)) skip = min(skip, max_skip) return min(skip, instances - 1) @@ -233,7 +271,7 @@ def select_kernels( if pct is not None and pct < min_total_time_pct: continue selected.append(item) - if len(selected) >= top_k: + if top_k > 0 and len(selected) >= top_k: break return selected @@ -285,7 +323,7 @@ def build_ncu_plan_json( "kernel_match": { "name_base": "demangled", "mode": "regex", - "pattern": _regex_exact_kernel_name(item.name), + "pattern": _regex_kernel_name(item.name), }, "launch_skip": skip, "launch_count": count, @@ -294,8 +332,11 @@ def build_ncu_plan_json( "selection": { "rank": index, "total_time_ns": item.total_time_ns, + "source_gpu_duration_ns": item.total_time_ns, "time_pct": pct, + "discovery_gpu_time_pct": pct, "instances": item.instances, + "source_launches": item.instances, "avg_time_ns": item.avg_time_ns, }, } @@ -328,19 +369,19 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--nsys-csv", required=True, type=Path, help="nsys cuda_gpu_kern_sum CSV") parser.add_argument("--out-discovery", type=Path, help="output kernel discovery JSON") parser.add_argument("--out-plan", type=Path, help="output NCU plan JSON") - parser.add_argument("--top-k", type=int, default=5, help="maximum selected kernels") - parser.add_argument("--min-total-time-pct", type=float, default=3.0, help="minimum GPU time percentage") + parser.add_argument("--top-k", type=int, default=5, help="maximum selected kernels; 0 selects all") + parser.add_argument("--min-total-time-pct", type=float, default=0.0, help="minimum GPU time percentage") parser.add_argument("--min-instances", type=int, default=1, help="minimum launch count") parser.add_argument("--launch-count", type=int, default=10, help="requested NCU launches per kernel") - parser.add_argument("--warmup-fraction", type=float, default=0.10, help="fraction of launches to skip") - parser.add_argument("--max-launch-skip", type=int, default=100, help="maximum generated launch skip") + parser.add_argument("--warmup-fraction", type=float, default=0.0, help="fraction of launches to skip") + parser.add_argument("--max-launch-skip", type=int, default=1, help="maximum generated launch skip") parser.add_argument("--metric-set", default="gpu_kernel_estimation", help="NCU metric-set label") parser.add_argument("--archive-ncu-report", action="store_true", help="request .ncu-rep archive retention") parser.add_argument("--print-plan", action="store_true", help="print generated plan to stdout") args = parser.parse_args(argv) - if args.top_k < 1: - parser.error("--top-k must be >= 1") + if args.top_k < 0: + parser.error("--top-k must be >= 0") if args.launch_count < 1: parser.error("--launch-count must be >= 1") if args.warmup_fraction < 0: @@ -374,7 +415,6 @@ def main(argv: list[str] | None = None) -> int: max_launch_skip=args.max_launch_skip, archive_ncu_report=args.archive_ncu_report, ) - if args.out_discovery: write_json(args.out_discovery, discovery) if args.out_plan: diff --git a/scripts/profiling/iter_ncu_plan_profiles.py b/scripts/profiling/iter_ncu_plan_profiles.py new file mode 100644 index 0000000..a822dd3 --- /dev/null +++ b/scripts/profiling/iter_ncu_plan_profiles.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +"""Emit NCU plan profiles as tab-separated rows for shell wrappers.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + + +def _profile_section(profile: dict[str, Any]) -> str: + section = profile.get("section") + if section: + return str(section) + + kernel_name = str(profile.get("kernel_name") or "") + if "build_pairlist" in kernel_name: + return "pairlist" + if "force_inter_cell" in kernel_name or "inter_cell" in kernel_name: + return "pme_real_inter" + if "force_intra_cell" in kernel_name or "intra_cell" in kernel_name: + return "pme_real_intra" + return "" + + +def _profile_pattern(profile: dict[str, Any]) -> str: + match = profile.get("kernel_match") + if isinstance(match, dict): + pattern = match.get("pattern") + if pattern: + return str(pattern) + return "" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--plan", required=True, type=Path, help="input ncu_plan.json") + args = parser.parse_args(argv) + + with args.plan.open(encoding="utf-8") as handle: + plan = json.load(handle) + + profiles = plan.get("profiles") + if not isinstance(profiles, list) or not profiles: + print(f"{args.plan} has no profiles", file=sys.stderr) + return 1 + + for profile in profiles: + if not isinstance(profile, dict): + continue + name = str(profile.get("name") or "").strip() + pattern = _profile_pattern(profile) + if not name or not pattern: + continue + section = _profile_section(profile) + launch_skip = int(profile.get("launch_skip") or 0) + launch_count = int(profile.get("launch_count") or 1) + print("\t".join([name, section, pattern, str(launch_skip), str(launch_count)])) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/tests/test_ncu_plan_generation.sh b/scripts/tests/test_ncu_plan_generation.sh index d96fdcb..9238c85 100644 --- a/scripts/tests/test_ncu_plan_generation.sh +++ b/scripts/tests/test_ncu_plan_generation.sh @@ -27,14 +27,21 @@ trap 'rm -rf "${TMP_DIR}"' EXIT --raw-dir-prefix ncu_auto \ -- ./app --input case.inp >/dev/null +"${PYTHON_BIN}" "${REPO_DIR}/scripts/profiling/iter_ncu_plan_profiles.py" \ + --plan "${TMP_DIR}/ncu_plan.json" > "${TMP_DIR}/ncu_plan_profiles.tsv" + test -f "${TMP_DIR}/kernel_discovery.json" test -f "${TMP_DIR}/ncu_plan.json" test -f "${TMP_DIR}/ncu_commands.json" +test -f "${TMP_DIR}/ncu_plan_profiles.tsv" jq -e ' .schema_version == 1 and .summary.kernel_count == 5 and .kernels[0].instances == 120 and + .kernels[0].source_launches == 120 and + .kernels[0].source_gpu_duration_ns == 6350000 and + .kernels[0].discovery_gpu_time_pct == 63.5 and (.kernels[0].name | contains("inter_cell")) ' "${TMP_DIR}/kernel_discovery.json" >/dev/null @@ -45,15 +52,59 @@ jq -e ' .profiles[0].launch_skip == 12 and .profiles[0].launch_count == 10 and .profiles[0].kernel_match.name_base == "demangled" and - (.profiles[0].kernel_match.pattern | startswith("regex:^")) and + .profiles[0].kernel_match.pattern == "regex:.*inter_cell.*" and + .profiles[1].kernel_match.pattern == "regex:.*intra_cell.*" and + .profiles[2].kernel_match.pattern == "regex:.*build_pairlist.*" and (.profiles[2].kernel_name | contains("build_pairlist")) ' "${TMP_DIR}/ncu_plan.json" >/dev/null +if jq -r '.profiles[].kernel_match.pattern' "${TMP_DIR}/ncu_plan.json" | grep -q '[[:space:]]'; then + echo "generated NCU regex patterns must not contain whitespace" >&2 + exit 1 +fi + if jq -e '.profiles[].kernel_name | contains("tiny_kernel")' "${TMP_DIR}/ncu_plan.json" >/dev/null; then echo "tiny low-impact kernel should not be selected" >&2 exit 1 fi +cat > "${TMP_DIR}/dominant_nsys.csv" <<'CSV' +CUDA Kernel Summary +"Time (%)","Total Time (ns)","Instances","Avg (ns)","Med (ns)","Min (ns)","Max (ns)","StdDev (ns)","Name" +97.5,"9,750,000",120,81250,81000,79000,90000,1200,"void kern_compute_force_nonbond_table_linear_univ__force_inter_cell(float*)" +1.5,"150,000",120,1250,1200,1000,2000,200,"void kern_compute_force_nonbond_table_linear_univ__force_intra_cell(float*)" +0.8,"80,000",12,6666,6500,6000,9000,600,"void kern_build_pairlist<4, 256>(float*)" +0.4,"40,000",6,6666,6500,6000,9000,600,"void kern_compute_energy_nonbond_table_linear_univ__energyforce_inter_cell(float*)" +0.2,"20,000",10,2000,1900,1800,2500,100,"void tiny_kernel()" +CSV + +"${PYTHON_BIN}" "${REPO_DIR}/scripts/profiling/generate_ncu_plan.py" \ + --nsys-csv "${TMP_DIR}/dominant_nsys.csv" \ + --out-plan "${TMP_DIR}/dominant_ncu_plan.json" \ + --top-k 3 \ + --launch-count 10 >/dev/null + +jq -e ' + (.profiles | length) == 3 and + .profiles[0].kernel_match.pattern == "regex:.*force_inter_cell.*" and + .profiles[0].launch_skip == 1 and + .profiles[0].selection.source_gpu_duration_ns == 9750000 and + .profiles[0].selection.discovery_gpu_time_pct == 97.5 and + .profiles[1].kernel_match.pattern == "regex:.*force_intra_cell.*" and + .profiles[2].kernel_match.pattern == "regex:.*build_pairlist.*" +' "${TMP_DIR}/dominant_ncu_plan.json" >/dev/null + +"${PYTHON_BIN}" "${REPO_DIR}/scripts/profiling/generate_ncu_plan.py" \ + --nsys-csv "${TMP_DIR}/dominant_nsys.csv" \ + --out-plan "${TMP_DIR}/all_ncu_plan.json" \ + --top-k 0 \ + --launch-count 10 >/dev/null + +jq -e ' + (.profiles | length) == 5 and + .profiles[3].kernel_match.pattern == "regex:.*energyforce_inter_cell.*" +' "${TMP_DIR}/all_ncu_plan.json" >/dev/null + jq -e ' .schema_version == 1 and .execution.profiler == "ncu" and @@ -66,4 +117,8 @@ jq -e ' (.commands[0].argv | index("./app") != null) ' "${TMP_DIR}/ncu_commands.json" >/dev/null +grep -q $'\tpme_real_inter\t' "${TMP_DIR}/ncu_plan_profiles.tsv" +grep -q $'\tpme_real_intra\t' "${TMP_DIR}/ncu_plan_profiles.tsv" +grep -q $'\tpairlist\t' "${TMP_DIR}/ncu_plan_profiles.tsv" + echo "ncu plan generation tests passed" From 129e188ac72bf089719408b9d87aa6169f69e365 Mon Sep 17 00:00:00 2001 From: Yoshifumi Nakamura Date: Mon, 27 Jul 2026 14:09:51 +0900 Subject: [PATCH 3/7] Preserve NSYS discovery metadata for GPU estimates Signed-off-by: Yoshifumi Nakamura --- programs/genesis/run.sh | 88 ++++++++++++++++++- .../gpu_kernel_ensemble_average.sh | 9 ++ .../gpu_kernel_lightgbm_v10.sh | 52 +++++++++++ .../section_packages/gpu_kernel_mlp_v15.sh | 52 +++++++++++ ..._estimation_gpu_kernel_ensemble_average.sh | 29 ++++++ 5 files changed, 227 insertions(+), 3 deletions(-) diff --git a/programs/genesis/run.sh b/programs/genesis/run.sh index bb95ae9..dc7d1a6 100644 --- a/programs/genesis/run.sh +++ b/programs/genesis/run.sh @@ -275,14 +275,18 @@ genesis_run_ncu_profile() { local launch_count="$5" local profiler_level="$6" local section_name="$7" - shift 7 + local discovery_metadata_json="$8" + shift 8 local archive_path="${resultsdir}/padata_${profile_slug}.tgz" local archive_rel_path="results/padata_${profile_slug}.tgz" + local metadata_path="${archive_path%.tgz}.metadata.json" + local metadata_rel_path="${archive_rel_path%.tgz}.metadata.json" local raw_dir="ncu_${profile_slug}" local profile_log="${resultsdir}/log_${header}_ncu_${profile_slug}.txt" local ncu_args local profile_status + local python_bin local old_profiler_args="${BK_PROFILER_ARGS:-}" local old_profiler_raw_csv="${BK_PROFILER_NCU_RAW_CSV:-}" local had_profiler_args=0 @@ -326,6 +330,58 @@ genesis_run_ncu_profile() { fi if [ -n "$section_name" ]; then + if [ -n "$discovery_metadata_json" ] && [ "$discovery_metadata_json" != "null" ]; then + python_bin=$(genesis_python_bin) + "$python_bin" - \ + "$metadata_path" \ + "$archive_rel_path" \ + "$section_name" \ + "$profile_name" \ + "$profile_slug" \ + "$kernel_regex" \ + "$launch_skip" \ + "$launch_count" \ + "$discovery_metadata_json" <<'PY' +import json +import sys +from pathlib import Path + +( + metadata_path, + archive_path, + section_name, + profile_name, + profile_slug, + kernel_regex, + launch_skip, + launch_count, + discovery_raw, +) = sys.argv[1:10] + +discovery = json.loads(discovery_raw) +if isinstance(discovery, dict) and not discovery.get("section"): + discovery["section"] = section_name + +payload = { + "schema_version": 1, + "kind": "gpu_kernel_profile_metadata", + "profiler": "ncu", + "section": section_name, + "profile_name": profile_name, + "profile_slug": profile_slug, + "artifact_path": archive_path, + "ncu": { + "kernel_regex": kernel_regex, + "launch_skip": int(launch_skip), + "launch_count": int(launch_count), + }, + "nsys_discovery": discovery, +} + +Path(metadata_path).write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") +PY + echo "GENESIS NCU profile metadata: ${metadata_rel_path}" >&2 + fi genesis_register_section_artifact "$section_name" "$archive_rel_path" fi } @@ -379,7 +435,7 @@ genesis_run_ncu_profiles() { profile_input=$(genesis_prepare_ncu_input "${profile_cmd[$last_index]}" "$profile_name" "$profile_slug" "$profile_key") profile_cmd[$last_index]="$profile_input" - genesis_run_ncu_profile "$profile_name" "$profile_slug" "$kernel_regex" "$launch_skip" "$launch_count" "$profiler_level" "$(genesis_profile_section_name "$profile_name")" "${profile_cmd[@]}" + genesis_run_ncu_profile "$profile_name" "$profile_slug" "$kernel_regex" "$launch_skip" "$launch_count" "$profiler_level" "$(genesis_profile_section_name "$profile_name")" "" "${profile_cmd[@]}" done } @@ -519,6 +575,7 @@ genesis_run_ncu_plan_profiles() { local profile_seen=0 local profile_rows=() local profile_row + local discovery_metadata_json python_bin=$(genesis_python_bin) mapfile -t profile_rows < <("$python_bin" "${SCRIPT_DIR}/scripts/profiling/iter_ncu_plan_profiles.py" --plan "$plan_json") @@ -538,7 +595,32 @@ genesis_run_ncu_plan_profiles() { fi profile_input=$(genesis_prepare_ncu_input "${profile_cmd[$last_index]}" "$profile_name" "$profile_slug" "$profile_key") profile_cmd[$last_index]="$profile_input" - genesis_run_ncu_profile "$profile_name" "$profile_slug" "$kernel_regex" "$launch_skip" "$launch_count" "$profiler_level" "$section_name" "${profile_cmd[@]}" + discovery_metadata_json=$("$python_bin" - "$plan_json" "$profile_name" "$section_name" <<'PY' +import json +import sys + +plan_path, profile_name, section_name = sys.argv[1:4] +with open(plan_path, encoding="utf-8") as handle: + plan = json.load(handle) +for profile in plan.get("profiles", []): + if profile.get("name") == profile_name: + discovery = dict(profile.get("selection") or {}) + discovery.update({ + "section": section_name, + "kernel_name": profile.get("kernel_name"), + "kernel_match": profile.get("kernel_match"), + "profile_name": profile.get("name"), + "launch_skip": profile.get("launch_skip"), + "launch_count": profile.get("launch_count"), + "metric_set": profile.get("metric_set"), + }) + print(json.dumps(discovery, separators=(",", ":"))) + break +else: + print("{}") +PY + ) + genesis_run_ncu_profile "$profile_name" "$profile_slug" "$kernel_regex" "$launch_skip" "$launch_count" "$profiler_level" "$section_name" "$discovery_metadata_json" "${profile_cmd[@]}" done if [ "$profile_seen" -eq 0 ]; then echo "GENESIS NCU plan has no executable profiles: ${plan_json}" >&2 diff --git a/scripts/estimation/section_packages/gpu_kernel_ensemble_average.sh b/scripts/estimation/section_packages/gpu_kernel_ensemble_average.sh index 0a36e0a..ac7745d 100644 --- a/scripts/estimation/section_packages/gpu_kernel_ensemble_average.sh +++ b/scripts/estimation/section_packages/gpu_kernel_ensemble_average.sh @@ -166,6 +166,7 @@ bk_section_package_transform_gpu_kernel_ensemble_average() { estimation_package: ($candidate.estimation_package // ""), predicted_time_ns: ($kernel.predicted_time_ns // null), time_ratio_predicted_over_source: $time_ratio, + nsys_discovery: ($candidate.metrics.nsys_discovery // null), source_metrics: ($kernel.source_metrics // {}), predicted_metrics: ($kernel.metrics // {}), metric_comparisons: ($kernel.metric_comparisons // []) @@ -205,6 +206,7 @@ bk_section_package_transform_gpu_kernel_ensemble_average() { predicted_time: ($candidate.metrics.sample_predicted_time // (if ($candidate.metrics.total_predicted_time_ns // null) != null then ($candidate.metrics.total_predicted_time_ns / 1000000000) else null end)), predicted_time_ns: ($candidate.metrics.total_predicted_time_ns // null) }, + nsys_discovery: ($candidate.metrics.nsys_discovery // null), artifacts: ($candidate.artifacts // []) } ) @@ -233,6 +235,7 @@ bk_section_package_transform_gpu_kernel_ensemble_average() { time_ratio_predicted_over_source: .time_ratio_predicted_over_source, predicted_time_ns: .predicted_time_ns, source_time_ns: .source_time_ns, + nsys_discovery: .nsys_discovery, source_gpu: .source_gpu, target_gpu: .target_gpu })) @@ -287,6 +290,7 @@ bk_section_package_transform_gpu_kernel_ensemble_average() { predicted_time_ns_total: (if ($predicted_times_ns | length) > 0 then ($predicted_times_ns | add) else null end), predicted_time_ns_mean: (if ($predicted_times_ns | length) > 0 then (($predicted_times_ns | add) / ($predicted_times_ns | length)) else null end), mean_time_ratio_predicted_over_source: (if ($ratios | length) > 0 then (($ratios | add) / ($ratios | length)) else null end), + nsys_discovery: ($package_group | map(.nsys_discovery // null) | map(select(. != null)) | .[0] // null), metric_comparisons: $metric_comparisons } ) @@ -355,6 +359,11 @@ bk_section_package_transform_gpu_kernel_ensemble_average() { kernel_names: $kernel_names, kernel_summaries: $kernel_summaries, kernel_candidate_ratios: $kernel_means, + nsys_discovery: ($usable | map(.metrics.nsys_discovery // null) | map(select(. != null)) | .[0] // null), + nsys_discovery_candidates: ($usable | map({ + estimation_package: (.estimation_package // ""), + nsys_discovery: (.metrics.nsys_discovery // null) + }) | map(select(.nsys_discovery != null))), app_gpu_section_time: $app_section_time, mean_time: (if $can_project_section then $output_time else null end), mean_time_ratio_predicted_over_source: $mean_ratio diff --git a/scripts/estimation/section_packages/gpu_kernel_lightgbm_v10.sh b/scripts/estimation/section_packages/gpu_kernel_lightgbm_v10.sh index 7d9fedb..1f5c878 100644 --- a/scripts/estimation/section_packages/gpu_kernel_lightgbm_v10.sh +++ b/scripts/estimation/section_packages/gpu_kernel_lightgbm_v10.sh @@ -316,6 +316,43 @@ _bk_gpu_lightgbm_resolve_section_ncu_archive() { printf '%s\n' "" } +_bk_gpu_lightgbm_resolve_section_metadata() { + local item_json="$1" + local section_name="$2" + local ncu_archive="$3" + local scoped_var + local value + local artifact_path + local metadata_path + + scoped_var=$(_bk_gpu_lightgbm_section_var "BK_GPU_LIGHTGBM_METADATA_JSON" "$section_name") + value=$(_bk_gpu_lightgbm_env_value "$scoped_var") + if [[ -n "$value" ]]; then + printf '%s\n' "$value" + return 0 + fi + + if [[ -n "${BK_GPU_LIGHTGBM_METADATA_JSON:-}" ]]; then + printf '%s\n' "$BK_GPU_LIGHTGBM_METADATA_JSON" + return 0 + fi + + artifact_path="${ncu_archive:-$(_bk_gpu_lightgbm_first_artifact_path "$item_json")}" + if [[ -n "$artifact_path" ]]; then + case "$artifact_path" in + *.tar.gz) metadata_path="${artifact_path%.tar.gz}.metadata.json" ;; + *.tgz) metadata_path="${artifact_path%.tgz}.metadata.json" ;; + *) metadata_path="${artifact_path}.metadata.json" ;; + esac + if [[ -f "$metadata_path" ]]; then + printf '%s\n' "$metadata_path" + return 0 + fi + fi + + printf '%s\n' "" +} + _bk_gpu_lightgbm_resolve_section_prediction_csv() { local item_json="$1" local section_name="$2" @@ -764,12 +801,20 @@ bk_section_package_transform_gpu_kernel_lightgbm_v10() { local selector_kind="" local selector_value="" local selector + local ncu_archive + local metadata_json_path="" + local metadata_json="{}" section_name=$(echo "$item_json" | jq -r '.name // "gpu_section"') selector=$(_bk_gpu_lightgbm_resolve_section_kernel_selector "$item_json" "$section_name") IFS=$'\t' read -r selector_kind selector_value <<< "$selector" prediction_csv=$(_bk_gpu_lightgbm_resolve_section_prediction_csv "$item_json" "$section_name") input_csv=$(_bk_gpu_lightgbm_resolve_section_input_csv "$item_json" "$section_name") + ncu_archive=$(_bk_gpu_lightgbm_resolve_section_ncu_archive "$item_json" "$section_name") + metadata_json_path=$(_bk_gpu_lightgbm_resolve_section_metadata "$item_json" "$section_name" "$ncu_archive") + if [[ -n "$metadata_json_path" && -f "$metadata_json_path" ]]; then + metadata_json=$(jq -c '.' "$metadata_json_path") + fi if [[ -z "$prediction_csv" ]]; then run_outputs=$(_bk_gpu_lightgbm_run_predictor "$item_json" "$section_name") @@ -784,6 +829,8 @@ bk_section_package_transform_gpu_kernel_lightgbm_v10() { --arg prediction_log "$prediction_log" \ --arg selector_kind "$selector_kind" \ --arg selector_value "$selector_value" \ + --arg metadata_json_path "$metadata_json_path" \ + --argjson discovery_metadata "$metadata_json" \ --argjson parsed "$parsed_json" ' def selector_matches($kind; $value): if $kind == "" or $value == "" then true @@ -853,6 +900,10 @@ bk_section_package_transform_gpu_kernel_lightgbm_v10() { matched_kernels: $matched_kernels, section_time_ratio_predicted_over_source: $time_ratio } + + (if ($discovery_metadata | length) > 0 then { + nsys_discovery: ($discovery_metadata.nsys_discovery // $discovery_metadata), + nsys_discovery_metadata_path: $metadata_json_path + } else {} end) ) } | if .fallback_used == null then del(.fallback_used) else . end @@ -861,6 +912,7 @@ bk_section_package_transform_gpu_kernel_lightgbm_v10() { + [{kind: "gpu_lightgbm_prediction_csv", path: $prediction_csv}] + (if $input_csv != "" then [{kind: "gpu_lightgbm_input_csv", path: $input_csv}] else [] end) + (if $prediction_log != "" then [{kind: "gpu_lightgbm_log", path: $prediction_log}] else [] end) + + (if $metadata_json_path != "" then [{kind: "gpu_kernel_profile_metadata", path: $metadata_json_path}] else [] end) ) ' } diff --git a/scripts/estimation/section_packages/gpu_kernel_mlp_v15.sh b/scripts/estimation/section_packages/gpu_kernel_mlp_v15.sh index 64ad3fb..d464782 100644 --- a/scripts/estimation/section_packages/gpu_kernel_mlp_v15.sh +++ b/scripts/estimation/section_packages/gpu_kernel_mlp_v15.sh @@ -305,6 +305,43 @@ _bk_gpu_mlp_resolve_section_ncu_archive() { printf '%s\n' "" } +_bk_gpu_mlp_resolve_section_metadata() { + local item_json="$1" + local section_name="$2" + local ncu_archive="$3" + local scoped_var + local value + local artifact_path + local metadata_path + + scoped_var=$(_bk_gpu_mlp_section_var "BK_GPU_MLP_METADATA_JSON" "$section_name") + value=$(_bk_gpu_mlp_env_value "$scoped_var") + if [[ -n "$value" ]]; then + printf '%s\n' "$value" + return 0 + fi + + if [[ -n "${BK_GPU_MLP_METADATA_JSON:-}" ]]; then + printf '%s\n' "$BK_GPU_MLP_METADATA_JSON" + return 0 + fi + + artifact_path="${ncu_archive:-$(_bk_gpu_mlp_first_artifact_path "$item_json")}" + if [[ -n "$artifact_path" ]]; then + case "$artifact_path" in + *.tar.gz) metadata_path="${artifact_path%.tar.gz}.metadata.json" ;; + *.tgz) metadata_path="${artifact_path%.tgz}.metadata.json" ;; + *) metadata_path="${artifact_path}.metadata.json" ;; + esac + if [[ -f "$metadata_path" ]]; then + printf '%s\n' "$metadata_path" + return 0 + fi + fi + + printf '%s\n' "" +} + _bk_gpu_mlp_resolve_section_prediction_csv() { local item_json="$1" local section_name="$2" @@ -774,12 +811,20 @@ bk_section_package_transform_gpu_kernel_mlp_v15() { local selector_kind="" local selector_value="" local selector + local ncu_archive + local metadata_json_path="" + local metadata_json="{}" section_name=$(echo "$item_json" | jq -r '.name // "gpu_section"') selector=$(_bk_gpu_mlp_resolve_section_kernel_selector "$item_json" "$section_name") IFS=$'\t' read -r selector_kind selector_value <<< "$selector" prediction_csv=$(_bk_gpu_mlp_resolve_section_prediction_csv "$item_json" "$section_name") input_csv=$(_bk_gpu_mlp_resolve_section_input_csv "$item_json" "$section_name") + ncu_archive=$(_bk_gpu_mlp_resolve_section_ncu_archive "$item_json" "$section_name") + metadata_json_path=$(_bk_gpu_mlp_resolve_section_metadata "$item_json" "$section_name" "$ncu_archive") + if [[ -n "$metadata_json_path" && -f "$metadata_json_path" ]]; then + metadata_json=$(jq -c '.' "$metadata_json_path") + fi if [[ -z "$prediction_csv" ]]; then run_outputs=$(_bk_gpu_mlp_run_predictor "$item_json" "$section_name") @@ -795,6 +840,8 @@ bk_section_package_transform_gpu_kernel_mlp_v15() { --arg selector_kind "$selector_kind" \ --arg selector_value "$selector_value" \ --arg scaling_method "$scaling_method" \ + --arg metadata_json_path "$metadata_json_path" \ + --argjson discovery_metadata "$metadata_json" \ --argjson parsed "$parsed_json" ' def selector_matches($kind; $value): if $kind == "" or $value == "" then true @@ -864,6 +911,10 @@ bk_section_package_transform_gpu_kernel_mlp_v15() { matched_kernels: $matched_kernels, section_time_ratio_predicted_over_source: $time_ratio } + + (if ($discovery_metadata | length) > 0 then { + nsys_discovery: ($discovery_metadata.nsys_discovery // $discovery_metadata), + nsys_discovery_metadata_path: $metadata_json_path + } else {} end) ) } | if .fallback_used == null then del(.fallback_used) else . end @@ -872,6 +923,7 @@ bk_section_package_transform_gpu_kernel_mlp_v15() { + [{kind: "gpu_mlp_prediction_csv", path: $prediction_csv}] + (if $input_csv != "" then [{kind: "gpu_mlp_input_csv", path: $input_csv}] else [] end) + (if $prediction_log != "" then [{kind: "gpu_mlp_log", path: $prediction_log}] else [] end) + + (if $metadata_json_path != "" then [{kind: "gpu_kernel_profile_metadata", path: $metadata_json_path}] else [] end) ) ' } diff --git a/scripts/tests/test_estimation_gpu_kernel_ensemble_average.sh b/scripts/tests/test_estimation_gpu_kernel_ensemble_average.sh index 25222f3..eabe89d 100644 --- a/scripts/tests/test_estimation_gpu_kernel_ensemble_average.sh +++ b/scripts/tests/test_estimation_gpu_kernel_ensemble_average.sh @@ -93,6 +93,30 @@ cat > "${TMP_DIR}/breakdown.json" <<'EOF' } EOF +mkdir -p "${TMP_DIR}/results" +cat > "${TMP_DIR}/results/padata0.metadata.json" <<'EOF' +{ + "schema_version": 1, + "kind": "gpu_kernel_profile_metadata", + "profiler": "ncu", + "section": "gpu_kernel_region", + "profile_name": "k001_kern_a", + "artifact_path": "results/padata0.tgz", + "ncu": { + "kernel_regex": "regex:.*kern_a.*", + "launch_skip": 1, + "launch_count": 10 + }, + "nsys_discovery": { + "section": "gpu_kernel_region", + "kernel_name": "kern_a", + "source_gpu_duration_ns": 123456, + "discovery_gpu_time_pct": 12.5, + "source_launches": 42 + } +} +EOF + pushd "${TMP_DIR}" >/dev/null source scripts/estimation/common.sh source scripts/estimation/packages/instrumented_app_sections_dummy.sh @@ -152,6 +176,7 @@ if ! echo "$transformed_single" | jq -e ' .sections[0].metrics.package_summaries[0].ncu_sample.kernel_count == 1 and .sections[0].metrics.package_summaries[0].ncu_sample.source_time_ns == 1000 and .sections[0].metrics.package_summaries[0].ncu_sample.predicted_time_ns == 1000 and + .sections[0].metrics.package_summaries[0].nsys_discovery.source_gpu_duration_ns == 123456 and .sections[0].metrics.package_summaries[1].estimation_package == "gpu_kernel_mlp_v15" and near(.sections[0].metrics.package_summaries[1].projected_section_time; 30) and near(.sections[0].metrics.package_summaries[1].time_ratio_predicted_over_source; 3) and @@ -165,6 +190,7 @@ if ! echo "$transformed_single" | jq -e ' .sections[0].metrics.kernel_summaries[0].package_summaries[0].source_time_ns_total == 1000 and .sections[0].metrics.kernel_summaries[0].package_summaries[0].predicted_time_ns_total == 1000 and near(.sections[0].metrics.kernel_summaries[0].package_summaries[0].mean_time_ratio_predicted_over_source; 1) and + .sections[0].metrics.kernel_summaries[0].package_summaries[0].nsys_discovery.section == "gpu_kernel_region" and (.sections[0].metrics.kernel_summaries[0].package_summaries[0].metric_comparisons | length >= 2) and (.sections[0].metrics.kernel_summaries[0].package_summaries[0].metric_comparisons | map(select(.name == "O-Memory Throughput [%]" and .source_value_mean == 25 and .predicted_value_mean == 50 and .ratio_predicted_over_source_mean == 2)) | length == 1) and .sections[0].metrics.kernel_summaries[0].package_summaries[1].estimation_package == "gpu_kernel_mlp_v15" and @@ -173,6 +199,9 @@ if ! echo "$transformed_single" | jq -e ' near(.sections[0].metrics.mean_time_ratio_predicted_over_source; 2) and .sections[0].metrics.unique_kernel_count == 1 and .sections[0].metrics.kernel_names == ["kern_a"] and + .sections[0].metrics.nsys_discovery.kernel_name == "kern_a" and + .sections[0].metrics.nsys_discovery.discovery_gpu_time_pct == 12.5 and + (.sections[0].metrics.nsys_discovery_candidates | length == 2) and (.sections[0].candidate_estimates | length == 2) and near(.sections[0].candidate_estimates[0].time; 10) and near(.sections[0].candidate_estimates[1].time; 30) and From ad0b7d744f23a4835b4d968e835b039040dc2a6e Mon Sep 17 00:00:00 2001 From: Yoshifumi Nakamura Date: Mon, 27 Jul 2026 14:15:26 +0900 Subject: [PATCH 4/7] Fix NCU plan profiler argument rendering Signed-off-by: Yoshifumi Nakamura --- scripts/profiling/render_ncu_plan_commands.py | 6 +++++- scripts/tests/test_ncu_plan_generation.sh | 3 +++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/profiling/render_ncu_plan_commands.py b/scripts/profiling/render_ncu_plan_commands.py index 06ee377..6569834 100644 --- a/scripts/profiling/render_ncu_plan_commands.py +++ b/scripts/profiling/render_ncu_plan_commands.py @@ -22,6 +22,10 @@ def _shell_join(args: list[str]) -> str: return " ".join(shlex.quote(str(item)) for item in args) +def _profiler_args_env(args: list[str]) -> str: + return " ".join(str(item) for item in args) + + def _truth(value: Any) -> str: return "true" if bool(value) else "false" @@ -61,7 +65,7 @@ def _command_record( archive_report = bool(profile.get("archive_ncu_report", False)) env = { - "BK_PROFILER_ARGS": _shell_join(profiler_args), + "BK_PROFILER_ARGS": _profiler_args_env(profiler_args), "BK_PROFILER_NCU_RAW_CSV": "true", "BK_PROFILER_ARCHIVE_NCU_REPORT": _truth(archive_report), } diff --git a/scripts/tests/test_ncu_plan_generation.sh b/scripts/tests/test_ncu_plan_generation.sh index 9238c85..31a78a2 100644 --- a/scripts/tests/test_ncu_plan_generation.sh +++ b/scripts/tests/test_ncu_plan_generation.sh @@ -112,6 +112,9 @@ jq -e ' (.commands | length) == 3 and .commands[0].env.BK_PROFILER_NCU_RAW_CSV == "true" and (.commands[0].env.BK_PROFILER_ARGS | contains("--kernel-name-base")) and + (.commands[0].env.BK_PROFILER_ARGS | contains("\u0027") | not) and + (.commands[0].env.BK_PROFILER_ARGS | contains("\"") | not) and + (.commands[0].env.BK_PROFILER_ARGS | contains("regex:.*inter_cell.*")) and (.commands[0].argv | index("bk_profiler") == 0) and (.commands[0].argv | index("--archive") != null) and (.commands[0].argv | index("./app") != null) From af233a89005731298ab162fd08578050c4e3ca90 Mon Sep 17 00:00:00 2001 From: Yoshifumi Nakamura Date: Mon, 27 Jul 2026 16:07:42 +0900 Subject: [PATCH 5/7] Handle read-only result artifacts in send job Signed-off-by: Yoshifumi Nakamura --- scripts/job_functions.sh | 10 ++++++---- .../result_server/process_and_send_results.sh | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+), 4 deletions(-) create mode 100644 scripts/result_server/process_and_send_results.sh diff --git a/scripts/job_functions.sh b/scripts/job_functions.sh index 8aa686e..2aeffc6 100644 --- a/scripts/job_functions.sh +++ b/scripts/job_functions.sh @@ -207,12 +207,13 @@ ${job_prefix}_send_results: environment: name: \$CI_COMMIT_BRANCH script: - - bash scripts/collect_timing.sh - - bash scripts/result.sh ${program} ${system} ${mode} ${build_job} ${run_job} \$CI_PIPELINE_ID - - bash scripts/result_server/send_results.sh + - id + - bash -lc 'ls -la results/ 2>&1 | head -20' + - stat results/ results/result0.json 2>&1 || true + - bash scripts/result_server/process_and_send_results.sh ${program} ${system} ${mode} ${build_job} ${run_job} \$CI_PIPELINE_ID artifacts: paths: - - results/ + - send_results_workspace/results/ expire_in: 1 week " >> "$output" @@ -291,6 +292,7 @@ ${job_prefix}_estimate: name: \$CI_COMMIT_BRANCH script: - echo \"Running estimation for ${code}\" + - if [ -d send_results_workspace/results ]; then cp -R send_results_workspace/results results; fi - bash scripts/estimation/run.sh ${code} artifacts: paths: diff --git a/scripts/result_server/process_and_send_results.sh b/scripts/result_server/process_and_send_results.sh new file mode 100644 index 0000000..1f95f73 --- /dev/null +++ b/scripts/result_server/process_and_send_results.sh @@ -0,0 +1,19 @@ +#!/bin/bash +set -euo pipefail + +if [ "$#" -ne 6 ]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +project_dir=$PWD +work_dir="${project_dir}/send_results_workspace" + +rm -rf "$work_dir" +mkdir -p "$work_dir" +cp -R results "$work_dir/results" + +cd "$work_dir" +bash "$project_dir/scripts/collect_timing.sh" +bash "$project_dir/scripts/result.sh" "$@" +bash "$project_dir/scripts/result_server/send_results.sh" From 5e00c6132f1333412a0e6789b65cba879f730869 Mon Sep 17 00:00:00 2001 From: Yoshifumi Nakamura Date: Mon, 27 Jul 2026 16:51:21 +0900 Subject: [PATCH 6/7] Normalize artifact permissions before upload Signed-off-by: Yoshifumi Nakamura --- scripts/matrix_generate.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/matrix_generate.sh b/scripts/matrix_generate.sh index d06add5..25d69e3 100644 --- a/scripts/matrix_generate.sh +++ b/scripts/matrix_generate.sh @@ -124,6 +124,7 @@ ${build_key}_build: - echo \"[BUILD] $program for $system\" - bash $program_path/build.sh $system - bash scripts/record_timestamp.sh results/build_end + - chmod -R a+rX artifacts results 2>/dev/null || true artifacts: paths: - artifacts/ @@ -153,6 +154,7 @@ ${job_prefix}_run: - bash $program_path/run.sh $system $nodes ${numproc_node} ${nthreads} - bash scripts/record_timestamp.sh results/run_end - echo \"Job completed\" + - chmod -R a+rX results 2>/dev/null || true - ls -la . # after_script: # - bash scripts/wait_for_nfs.sh results @@ -201,6 +203,7 @@ ${job_prefix}_build_run: - bash $program_path/run.sh $system $nodes ${numproc_node} ${nthreads} - bash scripts/record_timestamp.sh results/run_end - echo \"Job completed\" + - chmod -R a+rX artifacts results 2>/dev/null || true - ls -la . # after_script: # - bash scripts/wait_for_nfs.sh results From 061bb5ec0aa7207fb2ff9243963fb408bceb609f Mon Sep 17 00:00:00 2001 From: Yoshifumi Nakamura Date: Mon, 27 Jul 2026 17:32:16 +0900 Subject: [PATCH 7/7] Write estimation artifact archives outside results Signed-off-by: Yoshifumi Nakamura --- scripts/result_server/send_estimate.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/result_server/send_estimate.sh b/scripts/result_server/send_estimate.sh index 1f08708..9d20fdc 100644 --- a/scripts/result_server/send_estimate.sh +++ b/scripts/result_server/send_estimate.sh @@ -10,6 +10,7 @@ upload_estimation_artifacts() { local json_file="$1" local source_uuid="$2" local archive + local archive_dir local endpoint local endpoints local response @@ -25,7 +26,8 @@ upload_estimation_artifacts() { return 0 fi - archive="results/estimation_artifacts_${source_uuid}.tgz" + archive_dir=$(mktemp -d) + archive="${archive_dir}/estimation_artifacts_${source_uuid}.tgz" tar \ --exclude='*_prepare' \ --exclude='*_prepare/*' \ @@ -57,12 +59,12 @@ upload_estimation_artifacts() { if [[ -n "$response" ]]; then echo "$response" fi - rm -f "$archive" + rm -rf "$archive_dir" echo "Uploaded estimation artifacts for $json_file" return 0 fi - rm -f "$archive" + rm -rf "$archive_dir" if printf '%s\n' "$response" | grep -q '413'; then echo "WARNING: Skipping estimation artifact upload because the server rejected ${archive} as too large (HTTP 413)." >&2 echo "WARNING: Estimate JSON was already ingested; estimation artifacts remain available as GitLab artifacts." >&2