From 68f07237c6c1d5cc9968cf272f1a308c5eaa90f2 Mon Sep 17 00:00:00 2001 From: Mark Saroufim Date: Thu, 17 Sep 2026 14:51:17 -0700 Subject: [PATCH 1/3] Add Modal Nsight Compute profiling as the default CLI profile path --- .gitignore | 4 + README.md | 29 +++-- docs/linalg-qr-b200.md | 5 +- docs/modal-ncu-validation.md | 44 +++++++ docs/profiling.md | 92 ++++++++++---- src/cmd/mod.rs | 135 ++++++++++++++++++--- src/cmd/submit.rs | 22 +++- src/local.rs | 205 +++++++++++++++++++++++++++++++- src/service/mod.rs | 70 ++++++++++- templates/local_modal_runner.py | 101 +++++++++++++++- tests/test_modal_profile.py | 46 +++++++ 11 files changed, 673 insertions(+), 80 deletions(-) create mode 100644 docs/modal-ncu-validation.md create mode 100644 tests/test_modal_profile.py diff --git a/.gitignore b/.gitignore index 47e7497..1893e8a 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,7 @@ scratch.md .codex/ .popcorn/ .DS_Store + +popcorn-profile-*/ +validation/ +__pycache__/ diff --git a/README.md b/README.md index c0f6c08..648ac8e 100644 --- a/README.md +++ b/README.md @@ -7,39 +7,38 @@ Tested on linux and mac but should just work on Windows as well. ## New: Nsight Compute Profiling -Profile submissions on the hosted GPU Mode B200 Nsight Compute service. +Profile submissions on Modal (B200 by default). +Install and authenticate the Modal CLI first (`pip install modal && modal setup`). +`--profile` and `--mode profile` run in your Modal account, bypassing Popcorn +registration. Use `--profile-brev` to explicitly select the hosted Brev service. See [docs/profiling.md](docs/profiling.md) for a complete copy-paste flow. Quick QR v2 example: ```bash curl -O https://raw.githubusercontent.com/gpu-mode/reference-kernels/main/problems/linalg/qr_v2/submission.py -export POPCORN_BREV_PROFILER_URL=https://http--brev-profiler-proxy--dxfjds728w5v.code.run -popcorn submit submission.py --leaderboard qr_v2 --profile-brev --benchmark-index 0 --no-tui +popcorn submit submission.py --leaderboard qr_v2 --profile --benchmark-index 0 --no-tui ``` Quick `eigh` dense row example: ```bash curl -O https://raw.githubusercontent.com/gpu-mode/reference-kernels/main/problems/linalg/eigh_py/submission.py -export POPCORN_BREV_PROFILER_URL=https://http--brev-profiler-proxy--dxfjds728w5v.code.run -popcorn submit submission.py --leaderboard eigh --profile-brev --benchmark-index 3 --no-tui +popcorn submit submission.py --leaderboard eigh --profile --benchmark-index 3 --no-tui ``` Quick `cholesky` example: ```bash curl -O https://raw.githubusercontent.com/gpu-mode/reference-kernels/main/problems/linalg/cholesky_py/submission.py -export POPCORN_BREV_PROFILER_URL=https://http--brev-profiler-proxy--dxfjds728w5v.code.run -popcorn submit submission.py --leaderboard cholesky --profile-brev --benchmark-index 0 --no-tui +popcorn submit submission.py --leaderboard cholesky --profile --benchmark-index 0 --no-tui ``` The CLI downloads and extracts `ncu-details.txt` and `ncu-details.csv` for -agent-readable analysis. It also extracts the optional `.ncu-rep` GUI report and -ends with a macOS command that opens it in Nsight Compute: +agent-readable analysis. It also extracts the optional `.ncu-rep` GUI report which you can open in Nsight Compute: ```bash -open -a "NVIDIA Nsight Compute" profile.0-.../profile.ncu-rep +open -a "NVIDIA Nsight Compute" popcorn-profile-/profile-0/profile.ncu-rep ``` ## [NEW] Submit To The Linear Algebra Competition @@ -139,14 +138,14 @@ popcorn submit solution.py # Direct submission with all options popcorn submit --leaderboard grayscale_v2 --gpu A100 --mode leaderboard solution.py -# Nsight Compute profile on the hosted GPU Mode B200 profiler -POPCORN_BREV_PROFILER_URL=https://http--brev-profiler-proxy--dxfjds728w5v.code.run popcorn submit --leaderboard qr_v2 --profile-brev solution.py +# Nsight Compute profile on Modal, using your account (B200 by default) +popcorn submit --leaderboard qr_v2 --profile solution.py # Profile one QR v2 benchmark shape -POPCORN_BREV_PROFILER_URL=https://http--brev-profiler-proxy--dxfjds728w5v.code.run popcorn submit --leaderboard qr_v2 --profile-brev --benchmark-index 0 solution.py +popcorn submit --leaderboard qr_v2 --profile --benchmark-index 0 solution.py # Profile one eigh benchmark shape -POPCORN_BREV_PROFILER_URL=https://http--brev-profiler-proxy--dxfjds728w5v.code.run popcorn submit --leaderboard eigh --profile-brev --benchmark-index 3 solution.py +popcorn submit --leaderboard eigh --profile --benchmark-index 3 solution.py # Plain output mode (no TUI, good for CI/scripts) popcorn submit --no-tui --leaderboard grayscale_v2 --gpu A100 --mode test solution.py @@ -200,7 +199,7 @@ if either lookup fails, it stops instead of risking a stale cached image. - `test` - Quick test run to check correctness - `benchmark` - Benchmark your solution (no leaderboard impact) - `leaderboard` - Official ranked submission -- `profile` - Profile with Nsight Compute (limited availability) +- `profile` - Profile with Nsight Compute in your Modal account (default GPU: B200) ### Submissions diff --git a/docs/linalg-qr-b200.md b/docs/linalg-qr-b200.md index 47c7a1e..3c1063f 100644 --- a/docs/linalg-qr-b200.md +++ b/docs/linalg-qr-b200.md @@ -22,8 +22,9 @@ popcorn submit --leaderboard qr_v2 --gpu B200 --mode test submission.py Profile the first benchmark shape with Nsight Compute: ```bash -export POPCORN_BREV_PROFILER_URL=https://http--brev-profiler-proxy--dxfjds728w5v.code.run -popcorn submit --leaderboard qr_v2 --profile-brev --benchmark-index 0 submission.py +pip install modal +modal setup +popcorn submit --leaderboard qr_v2 --profile --benchmark-index 0 submission.py ``` The CLI downloads a `.zip` and extracts `ncu-details.txt`, `ncu-details.csv`, diff --git a/docs/modal-ncu-validation.md b/docs/modal-ncu-validation.md new file mode 100644 index 0000000..d18294e --- /dev/null +++ b/docs/modal-ncu-validation.md @@ -0,0 +1,44 @@ +# Modal NCU validation + +Validated on September 17, 2026, with no special Modal privileges or runtime flags. + +- Direct B200 matmul smoke: NCU 2025.2.1 captured one kernel in 39 passes and saved a report. +- CLI end-to-end: QR v2 benchmark 0 (`batch=20, n=32, cond=1, seed=43214`) on NVIDIA B200, using NCU 2026.2.0 and PyTorch 2.12.0+cu130. +- Resolved problem: `problems/linalg/qr_v2`; its evaluator supports profile mode with the `custom_kernel` NVTX range. +- `reference-kernels`: `51e22db671d36c1c76091c43c36a44546ba324a1`. +- `kernelbot`: `30ba5ce79107e5405b0cc1eda48ca551e7a51b16`. + +```bash +popcorn submit submission.py --profile --leaderboard qr_v2 \ + --benchmark-index 0 --ncu-kernel-name 'regex:geqr2' \ + --ncu-kernel-name-base demangled --ncu-launch-count 1 +``` + +The successful QR profile run produced an approximately 7.9 MiB `profile.ncu-rep`, `ncu-details.txt`, +`ncu-details.csv`, and a provenance manifest. The details export reopened the +report and identified `geqr2_batch_kernel_shmem` with measured GPU counters. +The matmul smoke reported six unavailable `ctc__*` metrics; successful capture +does not imply that every metric in `--set full` is populated. + +The installed release binary also passed with `--mode profile`, no explicit +`--gpu`, no kernel-name filter, and `--output summary.txt`. +It selected B200, saved all three report formats, and wrote the summary file. + +The change is contained in popcorn-cli; it uses KernelBot's task builder and +per-shape evaluator. The CLI overrides the NCU capture function in its ephemeral +worker to configure filters, child-process tracing, unlocked clocks, and detail +exports. It does not modify or deploy the hosted KernelBot service. + +`--profile` and `--mode profile` select Modal. `--profile-brev` explicitly selects +Brev. A failed Modal profile returns an error and never switches providers. +Omitting `--benchmark-index` preserves the task's entire `benchmarks` list; +selection and invalid indices are covered by CPU tests. + +Validation commands: + +```bash +cargo test # 58 tests passed +cargo clippy --all-targets -- -D warnings +python3 -m unittest discover -s tests -v # 3 tests passed +cargo build --release +``` diff --git a/docs/profiling.md b/docs/profiling.md index 1bc3e3c..57b0fa3 100644 --- a/docs/profiling.md +++ b/docs/profiling.md @@ -1,6 +1,6 @@ # Nsight Compute Profiling -This profiles GPU Mode submissions on the hosted B200 Nsight Compute service and +The default profiler runs GPU Mode submissions in your Modal account on B200 and downloads agent-readable `ncu-details.txt` / `ncu-details.csv` artifacts. The full `.ncu-rep` GUI report is still included for local inspection. @@ -8,23 +8,39 @@ The profiler uses the `benchmarks:` list from the active `reference-kernels` checkout. `--benchmark-index N` profiles `benchmarks[N]`; omitting `--benchmark-index` profiles every benchmark entry for that leaderboard. -## 1. Install and Register +## Supported problems -```bash -curl -fsSL https://raw.githubusercontent.com/gpu-mode/popcorn-cli/main/install.sh | bash -popcorn register discord -``` +The task evaluator must implement `profile` mode and launch the submission in +an NVTX push/pop range named `custom_kernel`. A PyTorch-profiler-only path is +not sufficient. QR, QR v2, Eigh, Cholesky, and the shared NVIDIA evaluator have +the required NCU path; only QR v2 has been tested end to end for this CLI change. +Check the actual evaluator selected by the task, since task-specific copies may +have different support. AMD and multi-GPU NCU profiling are unsupported. -Restart your terminal if `popcorn` is not found after installation. +Problem authors can follow the reference-kernels +[NCU integration guide](https://github.com/gpu-mode/reference-kernels/blob/docs/ncu-problem-guide/docs/ncu-profiling.md). -## 2. Set the Hosted Profiler URL +## 1. Install Popcorn and Modal ```bash -export POPCORN_BREV_PROFILER_URL=https://http--brev-profiler-proxy--dxfjds728w5v.code.run +curl -fsSL https://raw.githubusercontent.com/gpu-mode/popcorn-cli/main/install.sh | bash +pip install modal +modal setup ``` -`BREV_PROFILER_URL` is also accepted as a fallback, but -`POPCORN_BREV_PROFILER_URL` is preferred. +`--profile` and `--mode profile` use your own Modal +account and imply plain output. You do not need Popcorn registration or a +profiler URL. Modal bills this GPU run to your configured workspace. Use +`--gpu` to override B200, or put a GPU directive in the submission. + +## 2. Source versions and artifacts + +Each invocation resolves the latest `reference-kernels` and `kernelbot` main +commits. You can pin them with `POPCORN_REFERENCE_KERNELS_REF` and +`POPCORN_KERNELBOT_REF`. Every profile saves a `manifest.json` with both source +refs, the problem directory, GPU/system information, selected benchmark specs, +and NCU capture options alongside the reports in a unique `popcorn-profile-*` +directory. `--output` saves the text summary; the artifacts remain in that directory. ## 3. Profile QR v2 @@ -41,7 +57,7 @@ Profile one benchmark shape: ```bash popcorn submit submission.py \ --leaderboard qr_v2 \ - --profile-brev \ + --profile \ --benchmark-index 0 \ --no-tui ``` @@ -67,14 +83,13 @@ Profile the dense `n=512` leverage row: ```bash popcorn submit submission.py \ --leaderboard eigh \ - --profile-brev \ + --profile \ --benchmark-index 3 \ --no-tui ``` -The hosted profiler uses a deeper Nsight Compute launch window for `eigh` than -for QR v2 so PyTorch/cuSOLVER submissions can reach solver-path kernels after -clone/setup launches. +The default capture window is 10 kernel launches per benchmark. For late solver +kernels, select a kernel name or increase `--ncu-launch-count`. Current `eigh` benchmark index table from `reference-kernels` main `4a1153e`: @@ -110,31 +125,41 @@ Profile the `batch=4096, n=32` benchmark: ```bash popcorn submit submission.py \ --leaderboard cholesky \ - --profile-brev \ + --profile \ --benchmark-index 0 \ --no-tui ``` -The profiler will only profile the first 10 kernels, so if your solution uses more than that (the default Pytorch implementation does), then the profiler might be of limited use. +Capture late kernels with the same filters on any leaderboard: + +```bash +popcorn submit submission.py --leaderboard cholesky --profile --benchmark-index 0 \ + --ncu-kernel-name 'regex:my_kernel' --ncu-kernel-name-base demangled \ + --ncu-launch-count 2 +``` + +NCU follows child processes, captures the evaluator's `custom_kernel` NVTX range, +and leaves GPU clocks unchanged. Empty or failed captures return an error. +Multi-GPU profiling is unsupported. ## 6. Read the Details After the run finishes, the CLI downloads and extracts files like: ```text -profile.0-batch-20-n-32-cond-1-seed-43214.zip -profile.0-batch-20-n-32-cond-1-seed-43214/ncu-details.txt -profile.0-batch-20-n-32-cond-1-seed-43214/ncu-details.csv -profile.0-batch-20-n-32-cond-1-seed-43214/profile.ncu-rep # optional GUI report +popcorn-profile-/profile-0.zip +popcorn-profile-/profile-0/ncu-details.txt +popcorn-profile-/profile-0/ncu-details.csv +popcorn-profile-/profile-0/profile.ncu-rep # optional GUI report ``` Use `ncu-details.txt` or `ncu-details.csv` as the default artifact for AI -analysis. The CLI prints clickable links for these detail files. +analysis. The CLI prints local paths for the detail files and report. -The last line printed by the CLI opens the optional GUI report on macOS: +Open the GUI report on macOS: ```bash -open -a "NVIDIA Nsight Compute" 'profile.0-batch-20-n-32-cond-1-seed-43214/profile.ncu-rep' +open -a "NVIDIA Nsight Compute" 'popcorn-profile-/profile-0/profile.ncu-rep' ``` ## Profile All Benchmark Shapes @@ -144,7 +169,7 @@ Omit `--benchmark-index`: ```bash popcorn submit submission.py \ --leaderboard eigh \ - --profile-brev \ + --profile \ --no-tui ``` @@ -165,3 +190,18 @@ For leaderboard submission: ```bash popcorn submit submission.py --leaderboard qr_v2 --gpu B200 --mode leaderboard --no-tui ``` + +## Explicit Brev profiling + +Use `--profile-brev` to select the hosted Brev service explicitly. `--profile` +uses Modal only; errors never trigger a switch to Brev. Brev requires Popcorn +registration: + +```bash +popcorn register discord +export POPCORN_BREV_PROFILER_URL=https://http--brev-profiler-proxy--dxfjds728w5v.code.run +popcorn submit submission.py --leaderboard qr_v2 --profile-brev --benchmark-index 0 +``` + +`BREV_PROFILER_URL` is also accepted. Brev profiling uses the service's deployed +reference-kernels checkout; Modal source-ref overrides do not apply to it. diff --git a/src/cmd/mod.rs b/src/cmd/mod.rs index 39dfeb2..d3cf2cf 100644 --- a/src/cmd/mod.rs +++ b/src/cmd/mod.rs @@ -62,6 +62,10 @@ pub struct Cli { #[arg(long)] pub mode: Option, + /// Profile with Nsight Compute in your Modal account (default GPU: B200). + #[arg(long, conflicts_with = "profile_brev")] + pub profile: bool, + /// Profile on the hosted GPU Mode Brev B200 and save Nsight Compute artifacts locally. /// Requires POPCORN_BREV_PROFILER_URL, or BREV_PROFILER_URL as a fallback. #[arg(long)] @@ -72,9 +76,8 @@ pub struct Cli { #[arg(long, conflicts_with = "profile_brev")] pub local: bool, - /// Optional: Profile a single benchmark index when using --profile-brev - #[arg(long)] - pub benchmark_index: Option, + #[command(flatten)] + pub profile_options: crate::local::ProfileOptions, // Optional: Specify output file #[arg(short, long)] @@ -151,6 +154,10 @@ enum Commands { #[arg(long)] mode: Option, + /// Profile with Nsight Compute in your Modal account (default GPU: B200). + #[arg(long, conflicts_with = "profile_brev")] + profile: bool, + /// Profile on the hosted GPU Mode Brev B200 and save Nsight Compute artifacts locally. /// Requires POPCORN_BREV_PROFILER_URL, or BREV_PROFILER_URL as a fallback. #[arg(long)] @@ -161,9 +168,8 @@ enum Commands { #[arg(long, conflicts_with = "profile_brev")] local: bool, - /// Optional: Profile a single benchmark index when using --profile-brev - #[arg(long)] - benchmark_index: Option, + #[command(flatten)] + profile_options: crate::local::ProfileOptions, // Optional: Specify output file #[arg(short, long)] @@ -213,26 +219,45 @@ pub async fn execute(cli: Cli) -> Result<()> { leaderboard, mode, profile_brev, + profile, local, - benchmark_index, + profile_options, output, no_tui, }) => { // Use filepath from Submit command first, fallback to top-level filepath let final_filepath = filepath.or(cli.filepath); + let profile_brev = profile_brev || cli.profile_brev; + let profile = profile || cli.profile; + let local = local || cli.local; + if profile_brev && (profile || local) { + return Err(anyhow!( + "--profile-brev cannot be combined with --profile or --local" + )); + } + let profile_options = profile_options.merge(cli.profile_options); let final_gpu = if profile_brev { Some("B200_Brev".to_string()) } else { - gpu.clone() + gpu.clone().or(cli.gpu.clone()) }; - let final_mode = if profile_brev { + let final_mode = if profile_brev || profile { Some("profile".to_string()) } else { - mode.clone() + mode.clone().or(cli.mode.clone()) }; - if local { - submit::run_submit_local(final_filepath, gpu, leaderboard, mode, output).await + profile_options.validate(final_mode.as_deref())?; + if local || use_modal_profile(profile, profile_brev, final_mode.as_deref()) { + submit::run_submit_local( + final_filepath, + final_gpu, + leaderboard.or(cli.leaderboard), + final_mode, + profile_options, + output, + ) + .await } else { let config = load_config()?; let cli_id = config.cli_id.ok_or_else(|| { @@ -245,14 +270,14 @@ pub async fn execute(cli: Cli) -> Result<()> { ) })?; - if no_tui || profile_brev { + if no_tui || cli.no_tui || is_profile_mode(final_mode.as_deref()) { submit::run_submit_plain( final_filepath, // Resolved filepath final_gpu, // From Submit command leaderboard, // From Submit command final_mode, // From Submit command cli_id, - benchmark_index.or(cli.benchmark_index), + profile_options, output, // From Submit command ) .await @@ -318,6 +343,8 @@ pub async fn execute(cli: Cli) -> Result<()> { None => { // Check if any of the submission-related flags were used at the top level if !cli.profile_brev + && !cli.profile + && !is_profile_mode(cli.mode.as_deref()) && !cli.local && (cli.gpu.is_some() || cli.leaderboard.is_some() || cli.mode.is_some()) { @@ -329,12 +356,19 @@ pub async fn execute(cli: Cli) -> Result<()> { // Handle the case where only a filepath is provided (for backward compatibility) if let Some(top_level_filepath) = cli.filepath { - if cli.local { + let mode = if cli.profile || cli.profile_brev { + Some("profile".to_string()) + } else { + cli.mode + }; + cli.profile_options.validate(mode.as_deref())?; + if cli.local || use_modal_profile(cli.profile, cli.profile_brev, mode.as_deref()) { submit::run_submit_local( Some(top_level_filepath), cli.gpu, cli.leaderboard, - cli.mode, + mode, + cli.profile_options, cli.output, ) .await @@ -350,14 +384,18 @@ pub async fn execute(cli: Cli) -> Result<()> { ) })?; - if cli.profile_brev { + if cli.profile_brev || is_profile_mode(mode.as_deref()) { submit::run_submit_plain( Some(top_level_filepath), - Some("B200_Brev".to_string()), + if cli.profile_brev { + Some("B200_Brev".to_string()) + } else { + cli.gpu + }, cli.leaderboard, Some("profile".to_string()), cli_id, - cli.benchmark_index, + cli.profile_options, cli.output, ) .await @@ -382,3 +420,62 @@ pub async fn execute(cli: Cli) -> Result<()> { } } } + +fn use_modal_profile(profile: bool, brev: bool, mode: Option<&str>) -> bool { + !brev && (profile || is_profile_mode(mode)) +} + +fn is_profile_mode(mode: Option<&str>) -> bool { + mode.is_some_and(|mode| mode.eq_ignore_ascii_case("profile")) +} + +#[cfg(test)] +mod profile_tests { + use super::*; + + #[test] + fn profile_is_modal_and_brev_requires_its_explicit_flag() { + assert!(use_modal_profile(true, false, None)); + assert!(use_modal_profile(false, false, Some("profile"))); + assert!(use_modal_profile(false, false, Some("PROFILE"))); + assert!(!use_modal_profile(false, true, Some("profile"))); + assert!(!use_modal_profile(false, false, Some("benchmark"))); + } + + #[test] + fn profile_flags_accept_filters_at_both_entry_points() { + for prefix in [vec!["popcorn"], vec!["popcorn", "submit"]] { + for flag in ["--profile", "--profile-brev"] { + let mut args = prefix.clone(); + args.extend([ + "submission.py", + flag, + "--benchmark-index", + "3", + "--ncu-kernel-name", + "regex:custom", + "--ncu-launch-count", + "2", + ]); + assert!(Cli::try_parse_from(args).is_ok()); + } + } + assert!(Cli::try_parse_from([ + "popcorn", + "submit", + "submission.py", + "--profile", + "--profile-brev" + ]) + .is_err()); + assert!(Cli::try_parse_from([ + "popcorn", + "submit", + "submission.py", + "--profile", + "--ncu-kernel-name-base", + "invalid" + ]) + .is_err()); + } +} diff --git a/src/cmd/submit.rs b/src/cmd/submit.rs index ba5093e..60ca37d 100644 --- a/src/cmd/submit.rs +++ b/src/cmd/submit.rs @@ -687,7 +687,7 @@ pub async fn run_submit_plain( leaderboard: Option, mode: Option, cli_id: String, - benchmark_index: Option, + profile_options: crate::local::ProfileOptions, output: Option, ) -> Result<()> { let file_to_submit = match filepath { @@ -757,7 +757,7 @@ pub async fn run_submit_plain( &file_to_submit, &file_content, &final_leaderboard, - benchmark_index, + &profile_options, Some(Box::new(|msg| { eprintln!("{}", msg); })), @@ -813,9 +813,11 @@ pub async fn run_submit_local( gpu: Option, leaderboard: Option, mode: Option, + profile_options: crate::local::ProfileOptions, output: Option, ) -> Result<()> { - let file_to_submit = filepath.ok_or_else(|| anyhow!("File path is required with --local"))?; + let file_to_submit = + filepath.ok_or_else(|| anyhow!("File path is required with --local or --profile"))?; let submission_path = Path::new(&file_to_submit); if !submission_path.exists() { return Err(anyhow!("File not found: {}", file_to_submit)); @@ -833,8 +835,18 @@ pub async fn run_submit_local( )); } + let final_mode = mode.ok_or_else(|| { + anyhow!( + "Submission mode not specified. Use --mode test, benchmark, leaderboard, or profile" + ) + })?; let final_gpu = gpu .or_else(|| directives.gpus.first().cloned()) + .or_else(|| { + final_mode + .eq_ignore_ascii_case("profile") + .then(|| "B200".to_string()) + }) .ok_or_else(|| anyhow!("GPU not specified. Use --gpu or add a GPU directive"))?; let final_leaderboard = leaderboard .or_else(|| { @@ -843,9 +855,6 @@ pub async fn run_submit_local( .ok_or_else(|| { anyhow!("Leaderboard not specified. Use --leaderboard or add a leaderboard directive") })?; - let final_mode = mode.ok_or_else(|| { - anyhow!("Submission mode not specified. Use --mode test, benchmark, or leaderboard") - })?; eprintln!("Running public evaluation in your Modal account"); eprintln!("Leaderboard: {}", final_leaderboard); @@ -859,6 +868,7 @@ pub async fn run_submit_local( &final_leaderboard, &final_gpu, &final_mode, + &profile_options, ) .await?; diff --git a/src/local.rs b/src/local.rs index 4a62884..3ae9729 100644 --- a/src/local.rs +++ b/src/local.rs @@ -3,6 +3,8 @@ use std::path::Path; use std::process::Stdio; use anyhow::{anyhow, Context, Result}; +use base64::Engine; +use serde::Serialize; use serde_json::Value; use tokio::io::{AsyncBufReadExt, BufReader}; use tokio::process::Command; @@ -12,6 +14,50 @@ use crate::service; const LOCAL_RUNNER: &str = include_str!("../templates/local_modal_runner.py"); const RESULT_MARKER: &str = "POPCORN_LOCAL_RESULT="; +#[derive(clap::Args, Debug, Default, Clone, Serialize)] +pub struct ProfileOptions { + /// Profile benchmarks[N]; omit to profile all benchmark shapes. + #[arg(long)] + pub benchmark_index: Option, + /// Capture only matching NCU kernel names (supports regex: expressions). + #[arg(long)] + pub ncu_kernel_name: Option, + /// How NCU interprets kernel names. + #[arg(long, value_parser = ["function", "demangled", "mangled"])] + pub ncu_kernel_name_base: Option, + /// Maximum matching kernel launches to capture per benchmark (default: 10). + #[arg(long)] + pub ncu_launch_count: Option, +} + +impl ProfileOptions { + pub fn merge(self, fallback: Self) -> Self { + Self { + benchmark_index: self.benchmark_index.or(fallback.benchmark_index), + ncu_kernel_name: self.ncu_kernel_name.or(fallback.ncu_kernel_name), + ncu_kernel_name_base: self.ncu_kernel_name_base.or(fallback.ncu_kernel_name_base), + ncu_launch_count: self.ncu_launch_count.or(fallback.ncu_launch_count), + } + } + + pub fn validate(&self, mode: Option<&str>) -> Result<()> { + if self.ncu_launch_count == Some(0) { + return Err(anyhow!("--ncu-launch-count must be greater than zero")); + } + if (self.benchmark_index.is_some() + || self.ncu_kernel_name.is_some() + || self.ncu_kernel_name_base.is_some() + || self.ncu_launch_count.is_some()) + && !mode.is_some_and(|mode| mode.eq_ignore_ascii_case("profile")) + { + return Err(anyhow!( + "Profiling options require --profile, --profile-brev, or --mode profile" + )); + } + Ok(()) + } +} + fn gpu_names(gpu: &str) -> Result<(&'static str, &'static str)> { match gpu.to_ascii_lowercase().as_str() { "t4" => Ok(("T4", "T4")), @@ -20,6 +66,7 @@ fn gpu_names(gpu: &str) -> Result<(&'static str, &'static str)> { "a100" | "a100-80gb" => Ok(("A100", "A100-80GB")), "h100" | "h100!" => Ok(("H100", "H100!")), "b200" => Ok(("B200", "B200")), + "b200_brev" => Err(anyhow!("Brev profiling requires --profile-brev; --profile uses Modal only")), _ => Err(anyhow!( "GPU '{}' is not supported by local Modal mode. Supported GPUs: T4, L4, L4x4, A100, H100, B200", gpu @@ -173,22 +220,95 @@ fn format_local_result(payload: &Value) -> Result { Ok(sections.join("\n\n")) } +fn save_profile_result(payload: &Value, output_dir: &Path) -> Result { + let result = &payload["result"]; + if result["success"].as_bool() != Some(true) { + return Err(anyhow!("Modal profiling failed: {}", result["error"])); + } + let runs = result["runs"] + .as_object() + .filter(|runs| !runs.is_empty()) + .ok_or_else(|| anyhow!("Modal returned no profile runs"))?; + std::fs::create_dir_all(output_dir)?; + std::fs::write( + output_dir.join("manifest.json"), + serde_json::to_vec_pretty(&serde_json::json!({ + "leaderboard": payload["leaderboard"], + "gpu": payload["gpu"], + "system": result["system"], + "problem_directory": payload["problem_directory"], + "reference_kernels_ref": payload["reference_kernels_ref"], + "kernelbot_ref": payload["kernelbot_ref"], + "profile_options": payload["profile_options"], + "benchmark_specs": payload["benchmark_specs"], + }))?, + )?; + let mut lines = vec![format!( + "Modal Nsight Compute profile: {} on {}", + payload["leaderboard"].as_str().unwrap_or("unknown"), + payload["gpu"].as_str().unwrap_or("unknown") + )]; + let mut failures = Vec::new(); + for (index, (key, run)) in runs.iter().enumerate() { + if let Some(failure) = run_failure(run) { + failures.push(format!("{}: {}", key, failure)); + } + let Some(trace) = run.pointer("/profile/trace").and_then(Value::as_str) else { + failures.push(format!("{}: No NCU report was produced", key)); + continue; + }; + let bytes = base64::engine::general_purpose::STANDARD + .decode(trace) + .context("Modal returned an invalid profile archive")?; + // Use a locally generated filename, never a path from the remote response. + let zip_path = output_dir.join(format!("profile-{}.zip", index)); + std::fs::write(&zip_path, &bytes)?; + let extracted = service::extract_profile_artifacts(&zip_path, &bytes)?; + if extracted.reports.is_empty() { + failures.push(format!("{}: NCU archive contains no .ncu-rep report", key)); + } + lines.push(format!( + "{}: {}", + key, + run.pointer("/run/result/benchmark.0.spec") + .and_then(Value::as_str) + .unwrap_or("unknown benchmark") + )); + for path in extracted.details.iter().chain(extracted.reports.iter()) { + lines.push(format!(" {}", path.display())); + } + } + lines.push(format!( + "Manifest: {}", + output_dir.join("manifest.json").display() + )); + if !failures.is_empty() { + return Err(anyhow!("{}\n\n{}", lines.join("\n"), failures.join("\n"))); + } + Ok(lines.join("\n")) +} + pub async fn run_modal_submission( submission_path: &Path, leaderboard: &str, gpu: &str, mode: &str, + profile_options: &ProfileOptions, ) -> Result { if !matches!( mode.to_ascii_lowercase().as_str(), - "test" | "benchmark" | "leaderboard" + "test" | "benchmark" | "leaderboard" | "profile" ) { return Err(anyhow!( - "Local Modal mode supports test, benchmark, and leaderboard; got '{}'", + "Local Modal mode supports test, benchmark, leaderboard, and profile; got '{}'", mode )); } + profile_options.validate(Some(mode))?; let (kernelbot_gpu, modal_gpu) = gpu_names(gpu)?; + if mode.eq_ignore_ascii_case("profile") && modal_gpu.contains(':') { + return Err(anyhow!("Nsight Compute profiling requires a single GPU")); + } let helper = tempfile::Builder::new() .prefix("popcorn-local-modal-") .suffix(".py") @@ -208,6 +328,8 @@ pub async fn run_modal_submission( .env("POPCORN_LOCAL_GPU", kernelbot_gpu) .env("POPCORN_LOCAL_MODAL_GPU", modal_gpu) .env("POPCORN_LOCAL_MODE", mode.to_ascii_lowercase()) + .env("POPCORN_PROFILE_OPTIONS", serde_json::to_string(profile_options)?) + .kill_on_drop(true) .stdout(Stdio::piped()) .stderr(Stdio::piped()) .spawn() @@ -254,13 +376,90 @@ pub async fn run_modal_submission( } let payload = payload.ok_or_else(|| anyhow!("Modal returned no Popcorn result"))?; let payload: Value = serde_json::from_str(&payload).context("Modal returned invalid JSON")?; - format_local_result(&payload) + if mode.eq_ignore_ascii_case("profile") { + let output_dir = tempfile::Builder::new() + .prefix("popcorn-profile-") + .tempdir_in(std::env::current_dir()?)? + .into_path(); + save_profile_result(&payload, &output_dir) + } else { + format_local_result(&payload) + } } #[cfg(test)] mod tests { use super::*; + fn profile_payload(trace: &str) -> Value { + serde_json::json!({ + "leaderboard": "qr_v2", "gpu": "B200", "result": { + "success": true, "runs": {"profile.3": { + "run": {"success": true, "result": {"benchmark.0.spec": "n: 512"}}, + "profile": {"trace": trace} + }} + } + }) + } + + #[test] + fn extracts_profile_artifacts_and_records_provenance() { + use std::io::{Cursor, Write}; + let mut zip = zip::ZipWriter::new(Cursor::new(Vec::new())); + for name in ["profile.ncu-rep", "ncu-details.txt", "ncu-details.csv"] { + zip.start_file( + format!("profile_data/{}", name), + zip::write::SimpleFileOptions::default(), + ) + .unwrap(); + zip.write_all(b"captured data").unwrap(); + } + let trace = + base64::engine::general_purpose::STANDARD.encode(zip.finish().unwrap().into_inner()); + let mut payload = profile_payload(&trace); + payload["reference_kernels_ref"] = serde_json::json!("verified-sha"); + payload["profile_options"] = serde_json::json!({"benchmark_index": 3}); + let directory = tempfile::tempdir().unwrap(); + let output = save_profile_result(&payload, directory.path()).unwrap(); + assert!(output.contains("profile.3: n: 512")); + assert_eq!( + std::fs::read(directory.path().join("profile-0/profile.ncu-rep")).unwrap(), + b"captured data" + ); + assert!(output.contains("ncu-details.csv")); + let manifest = std::fs::read_to_string(directory.path().join("manifest.json")).unwrap(); + assert!(manifest.contains("verified-sha")); + assert!(manifest.contains("benchmark_index")); + } + + #[test] + fn rejects_missing_reports_and_failed_runs() { + let directory = tempfile::tempdir().unwrap(); + let mut payload = profile_payload(""); + payload["result"]["runs"]["profile.3"]["profile"] = Value::Null; + payload["result"]["runs"]["profile.3"]["run"]["success"] = Value::Bool(false); + payload["result"]["runs"]["profile.3"]["run"]["stderr"] = serde_json::json!("NCU denied"); + let error = save_profile_result(&payload, directory.path()) + .unwrap_err() + .to_string(); + assert!(error.contains("NCU denied")); + assert!(error.contains("No NCU report")); + payload["result"]["runs"] = serde_json::json!({}); + assert!(save_profile_result(&payload, directory.path()).is_err()); + } + + #[test] + fn validates_profile_options_before_launch() { + let mut options = ProfileOptions { + benchmark_index: Some(0), + ..Default::default() + }; + assert!(options.validate(Some("benchmark")).is_err()); + assert!(options.validate(Some("profile")).is_ok()); + options.ncu_launch_count = Some(0); + assert!(options.validate(Some("profile")).is_err()); + } + #[test] fn maps_popcorn_gpu_names_to_modal() { assert_eq!(gpu_names("B200").unwrap(), ("B200", "B200")); diff --git a/src/service/mod.rs b/src/service/mod.rs index d0c1ecc..b72c71d 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -651,7 +651,7 @@ pub async fn profile_brev_solution>( filepath: P, file_content: &[u8], leaderboard: &str, - benchmark_index: Option, + options: &crate::local::ProfileOptions, on_log: Option>, ) -> Result { let base_url = env::var("POPCORN_BREV_PROFILER_URL") @@ -663,6 +663,51 @@ pub async fn profile_brev_solution>( })?; let base_url = base_url.trim_end_matches('/'); + let requested_capture_options: Vec<&str> = [ + ("ncu_kernel_name", options.ncu_kernel_name.is_some()), + ( + "ncu_kernel_name_base", + options.ncu_kernel_name_base.is_some(), + ), + ("ncu_launch_count", options.ncu_launch_count.is_some()), + ] + .into_iter() + .filter_map(|(name, requested)| requested.then_some(name)) + .collect(); + if !requested_capture_options.is_empty() { + let health_resp = client + .get(format!("{}/health", base_url)) + .timeout(Duration::from_secs(30)) + .send() + .await?; + let health_status = health_resp.status(); + if !health_status.is_success() { + return Err(anyhow!( + "Profiler capability check returned status {}: {}", + health_status, + response_error_text(health_resp).await? + )); + } + let health: Value = health_resp.json().await?; + let supported = health + .pointer("/capabilities/request_capture_options") + .and_then(Value::as_array) + .ok_or_else(|| { + anyhow!( + "Profiler does not advertise request-scoped NCU capture; \ + refusing to silently run the default capture window" + ) + })?; + for name in &requested_capture_options { + if !supported.iter().any(|item| item.as_str() == Some(*name)) { + return Err(anyhow!( + "Profiler does not advertise request capture option {}", + name + )); + } + } + } + let filename = filepath .as_ref() .file_name() @@ -673,10 +718,20 @@ pub async fn profile_brev_solution>( let mut form = Form::new() .part("file", part) .text("leaderboard", leaderboard.to_string()); - if let Some(index) = benchmark_index { + if let Some(index) = options.benchmark_index { form = form.text("benchmark_index", index.to_string()); } + if let Some(value) = &options.ncu_kernel_name { + form = form.text("ncu_kernel_name", value.clone()); + } + if let Some(value) = &options.ncu_kernel_name_base { + form = form.text("ncu_kernel_name_base", value.clone()); + } + if let Some(value) = options.ncu_launch_count { + form = form.text("ncu_launch_count", value.to_string()); + } + let resp = client .post(format!("{}/profile", base_url)) .multipart(form) @@ -887,12 +942,15 @@ async fn download_profile_artifacts( } #[derive(Debug)] -struct ExtractedProfileArtifacts { - details: Vec, - reports: Vec, +pub(crate) struct ExtractedProfileArtifacts { + pub details: Vec, + pub reports: Vec, } -fn extract_profile_artifacts(zip_path: &Path, bytes: &[u8]) -> Result { +pub(crate) fn extract_profile_artifacts( + zip_path: &Path, + bytes: &[u8], +) -> Result { let mut archive = ZipArchive::new(Cursor::new(bytes)).map_err(|e| { anyhow!( "Failed to read profile artifact {}: {}", diff --git a/templates/local_modal_runner.py b/templates/local_modal_runner.py index 6157d14..81f1945 100644 --- a/templates/local_modal_runner.py +++ b/templates/local_modal_runner.py @@ -5,6 +5,9 @@ """ import dataclasses +import base64 +import functools +import subprocess import glob import json import os @@ -133,10 +136,20 @@ def _github_ref(repo: str, override_env: str) -> str: "/opt/cutlass/include:/opt/cutlass/tools/util/include" ), "PYTHONPATH": "/opt/kernelbot/src", + # The remote worker imports this module too. Carry only public build + # configuration across, never the user's local paths or credentials. + "POPCORN_REFERENCE_KERNELS_REF": reference_ref, + "POPCORN_KERNELBOT_REF": kernelbot_ref, + "POPCORN_LOCAL_MODAL_GPU": os.environ["POPCORN_LOCAL_MODAL_GPU"], + "POPCORN_LOCAL_MODE": os.environ["POPCORN_LOCAL_MODE"], } ) ) +# CUDA devel includes NCU. Verify the executable at image build time. +if os.environ.get("POPCORN_LOCAL_MODE") == "profile": + cuda_image = cuda_image.run_commands("ncu --version") + app = modal.App("popcorn-local-runner", image=cuda_image) modal_gpu = os.environ["POPCORN_LOCAL_MODAL_GPU"] @@ -166,11 +179,76 @@ def _find_problem(leaderboard: str) -> tuple[Path, list[str]]: return Path("/opt/reference-kernels/problems") / directory / "task.yml", supported_gpus +def _select_benchmarks(config: dict, options: dict) -> dict: + benchmarks = config.get("benchmarks", []) + if not benchmarks: + raise ValueError("This task has no benchmark shapes to profile") + index = options.get("benchmark_index") + if index is None: + return config + if not isinstance(index, int) or index < 0 or index >= len(benchmarks): + raise ValueError(f"Benchmark index {index} is out of range (0..{len(benchmarks) - 1})") + return {**config, "benchmarks": [benchmarks[index]]} + + +def _ncu_command(call: list[str], output_dir: Path, options: dict) -> list[str]: + count = options.get("ncu_launch_count") + if count is None: + count = 10 + if not isinstance(count, int) or count <= 0: + raise ValueError("NCU launch count must be a positive integer") + command = [ + "ncu", "--set", "full", "--target-processes", "all", + "--nvtx", "--nvtx-include", "custom_kernel/", + "--import-source", "1", "--launch-count", str(count), + "--cache-control", "all", "--clock-control", "none", + "--replay-mode", "kernel", "--force-overwrite", + "--export", str(output_dir / "profile.ncu-rep"), + ] + for key, flag in [("ncu_kernel_name", "--kernel-name"), + ("ncu_kernel_name_base", "--kernel-name-base")]: + if options.get(key): + command.extend([flag, options[key]]) + return command + ["--", *call] + + +def _profile_ncu(call, seed, timeout, multi_gpu, output_dir, *, options): + from libkernelbot.run_eval import ProfileResult, _directory_to_zip_bytes, run_program + + if multi_gpu: + raise ValueError("Nsight Compute profiling requires a single GPU") + result = run_program( + _ncu_command(call, output_dir, options), seed=seed, timeout=timeout, + multi_gpu=False, extra_env={"POPCORN_NCU": "1"}, + ) + report = output_dir / "profile.ncu-rep" + if not result.success or not report.is_file(): + result.success = False + result.stderr += "\nNsight Compute did not produce a report. Check NCU output and the evaluator's profile mode/NVTX range." + return result, None + for name, extra in [("ncu-details.txt", []), ("ncu-details.csv", ["--csv"])]: + details = subprocess.run( + ["ncu", "--import", str(report), "--page", "details", *extra], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=120, + ) + if details.returncode: + result.success = False + result.stderr += f"\nFailed to export {name}: {details.stderr}" + else: + (output_dir / name).write_text(details.stdout) + details_path = output_dir / "ncu-details.txt" + if details_path.exists(): + result.result["benchmark.0.report"] = base64.b64encode(details_path.read_bytes()).decode() + return result, ProfileResult( + profiler="Nsight-Compute", trace=_directory_to_zip_bytes(output_dir), download_url=None, + ) + + @app.function(gpu=modal_gpu, timeout=3600) -def evaluate(submission: str, leaderboard: str, gpu: str, mode: str) -> dict: +def evaluate(submission: str, leaderboard: str, gpu: str, mode: str, profile_options: dict) -> dict: try: from libkernelbot.consts import GPU_TO_SM, SubmissionMode - from libkernelbot.run_eval import run_config + from libkernelbot import run_eval from libkernelbot.task import build_task_config, make_task_definition task_path, supported_gpus = _find_problem(leaderboard) @@ -187,13 +265,29 @@ def evaluate(submission: str, leaderboard: str, gpu: str, mode: str) -> dict: arch=GPU_TO_SM[gpu], mode=SubmissionMode(mode), ) - result = run_config(config) + if mode == "profile": + config = _select_benchmarks(config, profile_options) + # Keep KernelBot's task setup, timeouts and per-shape evaluator execution. + # Only replace NCU capture to add Modal-safe clocks and CLI filters. + original_profile = run_eval.profile_program_ncu + try: + run_eval.profile_program_ncu = functools.partial(_profile_ncu, options=profile_options) + result = run_eval.run_config(config) + finally: + run_eval.profile_program_ncu = original_profile + index = profile_options.get("benchmark_index") + if index is not None and "profile.0" in result.runs: + result.runs[f"profile.{index}"] = result.runs.pop("profile.0") + else: + result = run_eval.run_config(config) return { "leaderboard": leaderboard, "problem_directory": str(task_path.parent.relative_to("/opt/reference-kernels/problems")), "gpu": gpu, "mode": mode, "ranking_by": definition.task.ranking_by.value, + "profile_options": profile_options, + "benchmark_specs": config.get("benchmarks", []), "reference_kernels_ref": reference_ref, "kernelbot_ref": kernelbot_ref, "result": dataclasses.asdict(result), @@ -222,5 +316,6 @@ def main(): os.environ["POPCORN_LOCAL_LEADERBOARD"], os.environ["POPCORN_LOCAL_GPU"], os.environ["POPCORN_LOCAL_MODE"], + json.loads(os.environ.get("POPCORN_PROFILE_OPTIONS", "{}")), ) print(RESULT_MARKER + json.dumps(payload, default=str, separators=(",", ":"))) diff --git a/tests/test_modal_profile.py b/tests/test_modal_profile.py new file mode 100644 index 0000000..e213322 --- /dev/null +++ b/tests/test_modal_profile.py @@ -0,0 +1,46 @@ +"""CPU checks for the embedded runner; importing the full module builds a Modal image.""" +import ast +import unittest +from pathlib import Path + +RUNNER = Path(__file__).resolve().parents[1] / "templates/local_modal_runner.py" +namespace = {"Path": Path} +tree = ast.parse(RUNNER.read_text()) +helpers = ast.Module(body=[node for node in tree.body if isinstance(node, ast.FunctionDef) + and node.name in {"_ncu_command", "_select_benchmarks"}], type_ignores=[]) +exec(compile(helpers, str(RUNNER), "exec"), namespace) + + +class ModalProfileTests(unittest.TestCase): + def test_selected_shape_and_all_shapes(self): + config = {"benchmarks": [{"n": 32}, {"n": 512}], "tests": [{"n": 16}]} + select = namespace["_select_benchmarks"] + self.assertIs(select(config, {}), config) + self.assertEqual(select(config, {"benchmark_index": 1})["benchmarks"], [{"n": 512}]) + self.assertEqual(len(config["benchmarks"]), 2) + self.assertEqual(select(config, {"benchmark_index": 1})["tests"], [{"n": 16}]) + for index in [-1, 2]: + with self.assertRaises(ValueError): + select(config, {"benchmark_index": index}) + with self.assertRaises(ValueError): + select({"benchmarks": []}, {}) + + def test_capture_tracks_child_processes_without_clock_control(self): + command = namespace["_ncu_command"](["python3", "eval.py"], Path("/tmp/out"), {}) + self.assertEqual(command[-3:], ["--", "python3", "eval.py"]) + for flag, value in [("--clock-control", "none"), ("--target-processes", "all"), + ("--nvtx-include", "custom_kernel/"), ("--launch-count", "10")]: + self.assertEqual(command[command.index(flag) + 1], value) + + def test_filters_remain_single_arguments(self): + command = namespace["_ncu_command"](["program"], Path("/tmp/out"), { + "ncu_kernel_name": "regex:late_kernel|kernel with spaces", + "ncu_kernel_name_base": "demangled", "ncu_launch_count": 2, + }) + self.assertEqual(command[command.index("--kernel-name") + 1], "regex:late_kernel|kernel with spaces") + self.assertEqual(command[command.index("--kernel-name-base") + 1], "demangled") + self.assertEqual(command[command.index("--launch-count") + 1], "2") + + +if __name__ == "__main__": + unittest.main() From 88ef1e136a53761c802b6d447846fc0047730c70 Mon Sep 17 00:00:00 2001 From: Mark Saroufim Date: Thu, 17 Sep 2026 14:52:06 -0700 Subject: [PATCH 2/3] Pin the NCU contributor guide link to its published commit --- docs/profiling.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/profiling.md b/docs/profiling.md index 57b0fa3..5994c69 100644 --- a/docs/profiling.md +++ b/docs/profiling.md @@ -18,7 +18,7 @@ Check the actual evaluator selected by the task, since task-specific copies may have different support. AMD and multi-GPU NCU profiling are unsupported. Problem authors can follow the reference-kernels -[NCU integration guide](https://github.com/gpu-mode/reference-kernels/blob/docs/ncu-problem-guide/docs/ncu-profiling.md). +[NCU integration guide](https://github.com/gpu-mode/reference-kernels/blob/bcfa2447fe77ca90456d46f41ff73d38bce6060a/docs/ncu-profiling.md). ## 1. Install Popcorn and Modal From 065a688be7a98cdb5c5274c7402fda606f8a4ccb Mon Sep 17 00:00:00 2001 From: Mark Saroufim Date: Thu, 17 Sep 2026 16:58:57 -0700 Subject: [PATCH 3/3] Route default profiling through authenticated KernelBot API --- README.md | 12 +- docs/linalg-qr-b200.md | 3 +- docs/modal-ncu-validation.md | 89 ++++++------ docs/profiling.md | 57 ++++---- src/cmd/mod.rs | 58 ++++---- src/cmd/submit.rs | 42 +++--- src/local.rs | 205 +-------------------------- src/main.rs | 1 + src/profile.rs | 240 ++++++++++++++++++++++++++++++++ src/service/mod.rs | 127 +++++++++++++++-- templates/local_modal_runner.py | 101 +------------- tests/test_modal_profile.py | 46 ------ 12 files changed, 505 insertions(+), 476 deletions(-) create mode 100644 src/profile.rs delete mode 100644 tests/test_modal_profile.py diff --git a/README.md b/README.md index 648ac8e..ab5042a 100644 --- a/README.md +++ b/README.md @@ -7,10 +7,10 @@ Tested on linux and mac but should just work on Windows as well. ## New: Nsight Compute Profiling -Profile submissions on Modal (B200 by default). -Install and authenticate the Modal CLI first (`pip install modal && modal setup`). -`--profile` and `--mode profile` run in your Modal account, bypassing Popcorn -registration. Use `--profile-brev` to explicitly select the hosted Brev service. +Profile submissions through GPU Mode (B200 by default). +Use your normal Popcorn registration. `--profile` and `--mode profile` submit +through GPU Mode and save NCU reports locally; no provider account or SDK is +needed. Use `--profile-brev` to explicitly select the Brev service. See [docs/profiling.md](docs/profiling.md) for a complete copy-paste flow. Quick QR v2 example: @@ -138,7 +138,7 @@ popcorn submit solution.py # Direct submission with all options popcorn submit --leaderboard grayscale_v2 --gpu A100 --mode leaderboard solution.py -# Nsight Compute profile on Modal, using your account (B200 by default) +# Nsight Compute profile through GPU Mode (B200 by default) popcorn submit --leaderboard qr_v2 --profile solution.py # Profile one QR v2 benchmark shape @@ -199,7 +199,7 @@ if either lookup fails, it stops instead of risking a stale cached image. - `test` - Quick test run to check correctness - `benchmark` - Benchmark your solution (no leaderboard impact) - `leaderboard` - Official ranked submission -- `profile` - Profile with Nsight Compute in your Modal account (default GPU: B200) +- `profile` - Profile with Nsight Compute through GPU Mode (default GPU: B200) ### Submissions diff --git a/docs/linalg-qr-b200.md b/docs/linalg-qr-b200.md index 3c1063f..eb519b1 100644 --- a/docs/linalg-qr-b200.md +++ b/docs/linalg-qr-b200.md @@ -22,8 +22,7 @@ popcorn submit --leaderboard qr_v2 --gpu B200 --mode test submission.py Profile the first benchmark shape with Nsight Compute: ```bash -pip install modal -modal setup +popcorn register discord popcorn submit --leaderboard qr_v2 --profile --benchmark-index 0 submission.py ``` diff --git a/docs/modal-ncu-validation.md b/docs/modal-ncu-validation.md index d18294e..a34b745 100644 --- a/docs/modal-ncu-validation.md +++ b/docs/modal-ncu-validation.md @@ -1,44 +1,45 @@ -# Modal NCU validation - -Validated on September 17, 2026, with no special Modal privileges or runtime flags. - -- Direct B200 matmul smoke: NCU 2025.2.1 captured one kernel in 39 passes and saved a report. -- CLI end-to-end: QR v2 benchmark 0 (`batch=20, n=32, cond=1, seed=43214`) on NVIDIA B200, using NCU 2026.2.0 and PyTorch 2.12.0+cu130. -- Resolved problem: `problems/linalg/qr_v2`; its evaluator supports profile mode with the `custom_kernel` NVTX range. -- `reference-kernels`: `51e22db671d36c1c76091c43c36a44546ba324a1`. -- `kernelbot`: `30ba5ce79107e5405b0cc1eda48ca551e7a51b16`. - -```bash -popcorn submit submission.py --profile --leaderboard qr_v2 \ - --benchmark-index 0 --ncu-kernel-name 'regex:geqr2' \ - --ncu-kernel-name-base demangled --ncu-launch-count 1 -``` - -The successful QR profile run produced an approximately 7.9 MiB `profile.ncu-rep`, `ncu-details.txt`, -`ncu-details.csv`, and a provenance manifest. The details export reopened the -report and identified `geqr2_batch_kernel_shmem` with measured GPU counters. -The matmul smoke reported six unavailable `ctc__*` metrics; successful capture -does not imply that every metric in `--set full` is populated. - -The installed release binary also passed with `--mode profile`, no explicit -`--gpu`, no kernel-name filter, and `--output summary.txt`. -It selected B200, saved all three report formats, and wrote the summary file. - -The change is contained in popcorn-cli; it uses KernelBot's task builder and -per-shape evaluator. The CLI overrides the NCU capture function in its ephemeral -worker to configure filters, child-process tracing, unlocked clocks, and detail -exports. It does not modify or deploy the hosted KernelBot service. - -`--profile` and `--mode profile` select Modal. `--profile-brev` explicitly selects -Brev. A failed Modal profile returns an error and never switches providers. -Omitting `--benchmark-index` preserves the task's entire `benchmarks` list; -selection and invalid indices are covered by CPU tests. - -Validation commands: - -```bash -cargo test # 58 tests passed -cargo clippy --all-targets -- -D warnings -python3 -m unittest discover -s tests -v # 3 tests passed -cargo build --release -``` +# Hosted NCU profiling validation + +`--profile` uses Popcorn authentication and the KernelBot API. KernelBot owns +Modal credentials and executes NCU; the client only uploads code and saves +returned artifacts. `--profile-brev` remains explicit. A capture error never +switches providers. + +The CLI calls `POST /profile/{leaderboard}/{gpu}` with multipart capture options. +This new endpoint requires the companion KernelBot update. Deploy the API and +GPU runner before releasing the CLI; older services do not support this route. +The existing `--local` evaluation workflow is independent of hosted profiling. + +Validation covers: + +- Rust tests for authenticated HTTP submission, multipart options, SSE errors, + artifact extraction, and malformed/missing captures. +- KernelBot tests for authenticated API requests, option validation, benchmark + selection, child-process capture, and report exports. +- An isolated integration run exercised the real FastAPI route, submission + preparation, KernelBackend, GPU `run_config`, and CLI artifact extraction, + with a fake database and an ephemeral B200 launcher. The CLI used normal + Popcorn header authentication with an empty `PATH` and no `MODAL_*` values. + It returned a 7,593,257-byte report plus text/CSV exports. +- NCU 2025.2.1 captured 39 passes. The exported report contained 322.78 us + duration, 12.51% achieved occupancy, 0.42% SM throughput, 70.09% L2 hit rate, + and 1,565,428 executed instructions. Six `ctc__*` metrics were unavailable. +- NCU 2026.2.0 from CUDA 13.3 produced many NaN counters on this Modal B200. + The server image therefore pins the tested 2025.2.1 version. The earlier + 2026.2 report was evidence of artifact delivery, not healthy counter coverage. + +GPU fixture provenance: `problems/linalg/qr_v2`, benchmark 0 +(`batch=20, n=32, cond=1, seed=43214`), reference-kernels +`51e22db671d36c1c76091c43c36a44546ba324a1` (the subsequent guide commit changes +only documentation). The initial KernelBot baseline was +`30ba5ce79107e5405b0cc1eda48ca551e7a51b16`. + +Only QR v2 has been tested end to end for this change. Other problems need an +NCU-compatible evaluator; accepting `profile` with only a PyTorch profiler path +is insufficient. Single-GPU NVIDIA capture is supported. Some requested metrics +can be unavailable on a particular GPU. + +The tested image used CUDA 13.3 and PyTorch 2.12.0+cu130, with +`regex:geqr2`, demangled kernel names, and launch count 1. The isolated test +is not a production deployment. [Operator validation run](https://modal.com/apps/coreauto/main/ap-p8eodYfHkZAMp0ZT5btR1y) +requires access to the operator's workspace. diff --git a/docs/profiling.md b/docs/profiling.md index 5994c69..b402e36 100644 --- a/docs/profiling.md +++ b/docs/profiling.md @@ -1,11 +1,11 @@ # Nsight Compute Profiling -The default profiler runs GPU Mode submissions in your Modal account on B200 and +The default profiler submits through GPU Mode on B200 and downloads agent-readable `ncu-details.txt` / `ncu-details.csv` artifacts. The full `.ncu-rep` GUI report is still included for local inspection. -The profiler uses the `benchmarks:` list from the active `reference-kernels` -checkout. `--benchmark-index N` profiles `benchmarks[N]`; omitting +The profiler uses the `benchmarks:` list from the task synced into the hosted +leaderboard configuration. `--benchmark-index N` profiles `benchmarks[N]`; omitting `--benchmark-index` profiles every benchmark entry for that leaderboard. ## Supported problems @@ -18,29 +18,34 @@ Check the actual evaluator selected by the task, since task-specific copies may have different support. AMD and multi-GPU NCU profiling are unsupported. Problem authors can follow the reference-kernels -[NCU integration guide](https://github.com/gpu-mode/reference-kernels/blob/bcfa2447fe77ca90456d46f41ff73d38bce6060a/docs/ncu-profiling.md). +[NCU integration guide](https://github.com/gpu-mode/reference-kernels/blob/a8044f1658acd4104558bedd3e78a8f096fd778a/docs/ncu-profiling.md). -## 1. Install Popcorn and Modal +## 1. Install and register ```bash curl -fsSL https://raw.githubusercontent.com/gpu-mode/popcorn-cli/main/install.sh | bash -pip install modal -modal setup +popcorn register discord ``` -`--profile` and `--mode profile` use your own Modal -account and imply plain output. You do not need Popcorn registration or a -profiler URL. Modal bills this GPU run to your configured workspace. Use -`--gpu` to override B200, or put a GPU directive in the submission. +`--profile` and `--mode profile` use the normal authenticated GPU Mode API and +imply plain output. No Modal installation, provider account, provider token, +or profiling URL is needed. The service owns the compute credentials. B200 is +the default GPU; use `--gpu` or a submission GPU directive to select another +supported GPU. The existing `--local` evaluation option is a separate workflow +for users deliberately choosing their own compute account. + +## 2. Benchmark selection and artifacts -## 2. Source versions and artifacts +The hosted service uses the task/evaluator already synced to that leaderboard. +Local reference-kernels edits and `POPCORN_REFERENCE_KERNELS_REF` do not change +a hosted run. Problem authors should ask an operator to sync their published +revision before validating it. -Each invocation resolves the latest `reference-kernels` and `kernelbot` main -commits. You can pin them with `POPCORN_REFERENCE_KERNELS_REF` and -`POPCORN_KERNELBOT_REF`. Every profile saves a `manifest.json` with both source -refs, the problem directory, GPU/system information, selected benchmark specs, -and NCU capture options alongside the reports in a unique `popcorn-profile-*` -directory. `--output` saves the text summary; the artifacts remain in that directory. +Each profile saves a `manifest.json` containing the leaderboard, GPU/system +information, selected benchmark specs, capture options, and an evaluation-config +SHA-256 digest. Reports go into a unique `popcorn-profile-*` directory. +`--output` saves the text summary; artifacts remain in that directory. The +config digest identifies the evaluated content; it is not a repository commit. ## 3. Profile QR v2 @@ -147,10 +152,10 @@ Multi-GPU profiling is unsupported. After the run finishes, the CLI downloads and extracts files like: ```text -popcorn-profile-/profile-0.zip -popcorn-profile-/profile-0/ncu-details.txt -popcorn-profile-/profile-0/ncu-details.csv -popcorn-profile-/profile-0/profile.ncu-rep # optional GUI report +popcorn-profile-/result-0/profile-0.zip +popcorn-profile-/result-0/profile-0/ncu-details.txt +popcorn-profile-/result-0/profile-0/ncu-details.csv +popcorn-profile-/result-0/profile-0/profile.ncu-rep # optional GUI report ``` Use `ncu-details.txt` or `ncu-details.csv` as the default artifact for AI @@ -159,7 +164,7 @@ analysis. The CLI prints local paths for the detail files and report. Open the GUI report on macOS: ```bash -open -a "NVIDIA Nsight Compute" 'popcorn-profile-/profile-0/profile.ncu-rep' +open -a "NVIDIA Nsight Compute" 'popcorn-profile-/result-0/profile-0/profile.ncu-rep' ``` ## Profile All Benchmark Shapes @@ -194,7 +199,7 @@ popcorn submit submission.py --leaderboard qr_v2 --gpu B200 --mode leaderboard - ## Explicit Brev profiling Use `--profile-brev` to select the hosted Brev service explicitly. `--profile` -uses Modal only; errors never trigger a switch to Brev. Brev requires Popcorn +uses the GPU Mode API; errors never trigger a switch to Brev. Brev requires Popcorn registration: ```bash @@ -203,5 +208,5 @@ export POPCORN_BREV_PROFILER_URL=https://http--brev-profiler-proxy--dxfjds728w5v popcorn submit submission.py --leaderboard qr_v2 --profile-brev --benchmark-index 0 ``` -`BREV_PROFILER_URL` is also accepted. Brev profiling uses the service's deployed -reference-kernels checkout; Modal source-ref overrides do not apply to it. +`BREV_PROFILER_URL` is also accepted. Brev profiling uses that service's deployed +reference-kernels checkout. diff --git a/src/cmd/mod.rs b/src/cmd/mod.rs index d3cf2cf..63ad695 100644 --- a/src/cmd/mod.rs +++ b/src/cmd/mod.rs @@ -62,7 +62,7 @@ pub struct Cli { #[arg(long)] pub mode: Option, - /// Profile with Nsight Compute in your Modal account (default GPU: B200). + /// Profile with Nsight Compute through GPU Mode (default GPU: B200). #[arg(long, conflicts_with = "profile_brev")] pub profile: bool, @@ -77,7 +77,7 @@ pub struct Cli { pub local: bool, #[command(flatten)] - pub profile_options: crate::local::ProfileOptions, + pub profile_options: crate::profile::ProfileOptions, // Optional: Specify output file #[arg(short, long)] @@ -154,7 +154,7 @@ enum Commands { #[arg(long)] mode: Option, - /// Profile with Nsight Compute in your Modal account (default GPU: B200). + /// Profile with Nsight Compute through GPU Mode (default GPU: B200). #[arg(long, conflicts_with = "profile_brev")] profile: bool, @@ -169,7 +169,7 @@ enum Commands { local: bool, #[command(flatten)] - profile_options: crate::local::ProfileOptions, + profile_options: crate::profile::ProfileOptions, // Optional: Specify output file #[arg(short, long)] @@ -230,9 +230,9 @@ pub async fn execute(cli: Cli) -> Result<()> { let profile_brev = profile_brev || cli.profile_brev; let profile = profile || cli.profile; let local = local || cli.local; - if profile_brev && (profile || local) { + if (profile_brev && (profile || local)) || (local && profile) { return Err(anyhow!( - "--profile-brev cannot be combined with --profile or --local" + "Profiling uses the hosted service; --local cannot be combined with profiling, and --profile conflicts with --profile-brev" )); } let profile_options = profile_options.merge(cli.profile_options); @@ -248,13 +248,15 @@ pub async fn execute(cli: Cli) -> Result<()> { }; profile_options.validate(final_mode.as_deref())?; - if local || use_modal_profile(profile, profile_brev, final_mode.as_deref()) { + if local && is_profile_mode(final_mode.as_deref()) { + return Err(anyhow!("Profiling uses the hosted service; omit --local")); + } + if local { submit::run_submit_local( final_filepath, final_gpu, leaderboard.or(cli.leaderboard), final_mode, - profile_options, output, ) .await @@ -274,10 +276,11 @@ pub async fn execute(cli: Cli) -> Result<()> { submit::run_submit_plain( final_filepath, // Resolved filepath final_gpu, // From Submit command - leaderboard, // From Submit command - final_mode, // From Submit command + leaderboard.or(cli.leaderboard), + final_mode, // From Submit command cli_id, profile_options, + profile_brev, output, // From Submit command ) .await @@ -285,8 +288,8 @@ pub async fn execute(cli: Cli) -> Result<()> { submit::run_submit_tui( final_filepath, // Resolved filepath final_gpu, // From Submit command - leaderboard, // From Submit command - final_mode, // From Submit command + leaderboard.or(cli.leaderboard), + final_mode, // From Submit command cli_id, output, // From Submit command ) @@ -362,13 +365,15 @@ pub async fn execute(cli: Cli) -> Result<()> { cli.mode }; cli.profile_options.validate(mode.as_deref())?; - if cli.local || use_modal_profile(cli.profile, cli.profile_brev, mode.as_deref()) { + if cli.local && is_profile_mode(mode.as_deref()) { + return Err(anyhow!("Profiling uses the hosted service; omit --local")); + } + if cli.local { submit::run_submit_local( Some(top_level_filepath), cli.gpu, cli.leaderboard, mode, - cli.profile_options, cli.output, ) .await @@ -396,6 +401,7 @@ pub async fn execute(cli: Cli) -> Result<()> { Some("profile".to_string()), cli_id, cli.profile_options, + cli.profile_brev, cli.output, ) .await @@ -421,10 +427,6 @@ pub async fn execute(cli: Cli) -> Result<()> { } } -fn use_modal_profile(profile: bool, brev: bool, mode: Option<&str>) -> bool { - !brev && (profile || is_profile_mode(mode)) -} - fn is_profile_mode(mode: Option<&str>) -> bool { mode.is_some_and(|mode| mode.eq_ignore_ascii_case("profile")) } @@ -433,13 +435,19 @@ fn is_profile_mode(mode: Option<&str>) -> bool { mod profile_tests { use super::*; - #[test] - fn profile_is_modal_and_brev_requires_its_explicit_flag() { - assert!(use_modal_profile(true, false, None)); - assert!(use_modal_profile(false, false, Some("profile"))); - assert!(use_modal_profile(false, false, Some("PROFILE"))); - assert!(!use_modal_profile(false, true, Some("profile"))); - assert!(!use_modal_profile(false, false, Some("benchmark"))); + #[tokio::test] + async fn local_rejects_profile_mode_inherited_from_top_level() { + let cli = Cli::try_parse_from([ + "popcorn", + "--mode", + "profile", + "submit", + "submission.py", + "--local", + ]) + .unwrap(); + let error = execute(cli).await.unwrap_err().to_string(); + assert!(error.contains("omit --local")); } #[test] diff --git a/src/cmd/submit.rs b/src/cmd/submit.rs index 60ca37d..bcc4582 100644 --- a/src/cmd/submit.rs +++ b/src/cmd/submit.rs @@ -681,13 +681,15 @@ pub async fn run_submit_tui( Ok(()) } +#[allow(clippy::too_many_arguments)] pub async fn run_submit_plain( filepath: Option, gpu: Option, leaderboard: Option, mode: Option, cli_id: String, - profile_options: crate::local::ProfileOptions, + profile_options: crate::profile::ProfileOptions, + profile_brev: bool, output: Option, ) -> Result<()> { let file_to_submit = match filepath { @@ -718,6 +720,11 @@ pub async fn run_submit_plain( None } }) + .or_else(|| { + mode.as_deref() + .is_some_and(|m| m.eq_ignore_ascii_case("profile")) + .then(|| "B200".to_string()) + }) .ok_or_else(|| anyhow!("GPU not specified. Use --gpu flag or add GPU directive to file"))?; let final_leaderboard = leaderboard @@ -749,9 +756,7 @@ pub async fn run_submit_plain( // Create client and submit let client = service::create_client(Some(cli_id))?; - let result = if final_mode.eq_ignore_ascii_case("profile") - && final_gpu.eq_ignore_ascii_case("B200_Brev") - { + let result = if final_mode.eq_ignore_ascii_case("profile") && profile_brev { service::profile_brev_solution( &client, &file_to_submit, @@ -763,6 +768,17 @@ pub async fn run_submit_plain( })), ) .await? + } else if final_mode.eq_ignore_ascii_case("profile") { + service::profile_solution( + &client, + &file_to_submit, + &file_content, + &final_leaderboard, + &final_gpu, + &profile_options, + Some(Box::new(|msg| eprintln!("{}", msg))), + ) + .await? } else { service::submit_solution( &client, @@ -813,11 +829,9 @@ pub async fn run_submit_local( gpu: Option, leaderboard: Option, mode: Option, - profile_options: crate::local::ProfileOptions, output: Option, ) -> Result<()> { - let file_to_submit = - filepath.ok_or_else(|| anyhow!("File path is required with --local or --profile"))?; + let file_to_submit = filepath.ok_or_else(|| anyhow!("File path is required with --local"))?; let submission_path = Path::new(&file_to_submit); if !submission_path.exists() { return Err(anyhow!("File not found: {}", file_to_submit)); @@ -835,18 +849,8 @@ pub async fn run_submit_local( )); } - let final_mode = mode.ok_or_else(|| { - anyhow!( - "Submission mode not specified. Use --mode test, benchmark, leaderboard, or profile" - ) - })?; let final_gpu = gpu .or_else(|| directives.gpus.first().cloned()) - .or_else(|| { - final_mode - .eq_ignore_ascii_case("profile") - .then(|| "B200".to_string()) - }) .ok_or_else(|| anyhow!("GPU not specified. Use --gpu or add a GPU directive"))?; let final_leaderboard = leaderboard .or_else(|| { @@ -855,6 +859,9 @@ pub async fn run_submit_local( .ok_or_else(|| { anyhow!("Leaderboard not specified. Use --leaderboard or add a leaderboard directive") })?; + let final_mode = mode.ok_or_else(|| { + anyhow!("Submission mode not specified. Use --mode test, benchmark, or leaderboard") + })?; eprintln!("Running public evaluation in your Modal account"); eprintln!("Leaderboard: {}", final_leaderboard); @@ -868,7 +875,6 @@ pub async fn run_submit_local( &final_leaderboard, &final_gpu, &final_mode, - &profile_options, ) .await?; diff --git a/src/local.rs b/src/local.rs index 3ae9729..4a62884 100644 --- a/src/local.rs +++ b/src/local.rs @@ -3,8 +3,6 @@ use std::path::Path; use std::process::Stdio; use anyhow::{anyhow, Context, Result}; -use base64::Engine; -use serde::Serialize; use serde_json::Value; use tokio::io::{AsyncBufReadExt, BufReader}; use tokio::process::Command; @@ -14,50 +12,6 @@ use crate::service; const LOCAL_RUNNER: &str = include_str!("../templates/local_modal_runner.py"); const RESULT_MARKER: &str = "POPCORN_LOCAL_RESULT="; -#[derive(clap::Args, Debug, Default, Clone, Serialize)] -pub struct ProfileOptions { - /// Profile benchmarks[N]; omit to profile all benchmark shapes. - #[arg(long)] - pub benchmark_index: Option, - /// Capture only matching NCU kernel names (supports regex: expressions). - #[arg(long)] - pub ncu_kernel_name: Option, - /// How NCU interprets kernel names. - #[arg(long, value_parser = ["function", "demangled", "mangled"])] - pub ncu_kernel_name_base: Option, - /// Maximum matching kernel launches to capture per benchmark (default: 10). - #[arg(long)] - pub ncu_launch_count: Option, -} - -impl ProfileOptions { - pub fn merge(self, fallback: Self) -> Self { - Self { - benchmark_index: self.benchmark_index.or(fallback.benchmark_index), - ncu_kernel_name: self.ncu_kernel_name.or(fallback.ncu_kernel_name), - ncu_kernel_name_base: self.ncu_kernel_name_base.or(fallback.ncu_kernel_name_base), - ncu_launch_count: self.ncu_launch_count.or(fallback.ncu_launch_count), - } - } - - pub fn validate(&self, mode: Option<&str>) -> Result<()> { - if self.ncu_launch_count == Some(0) { - return Err(anyhow!("--ncu-launch-count must be greater than zero")); - } - if (self.benchmark_index.is_some() - || self.ncu_kernel_name.is_some() - || self.ncu_kernel_name_base.is_some() - || self.ncu_launch_count.is_some()) - && !mode.is_some_and(|mode| mode.eq_ignore_ascii_case("profile")) - { - return Err(anyhow!( - "Profiling options require --profile, --profile-brev, or --mode profile" - )); - } - Ok(()) - } -} - fn gpu_names(gpu: &str) -> Result<(&'static str, &'static str)> { match gpu.to_ascii_lowercase().as_str() { "t4" => Ok(("T4", "T4")), @@ -66,7 +20,6 @@ fn gpu_names(gpu: &str) -> Result<(&'static str, &'static str)> { "a100" | "a100-80gb" => Ok(("A100", "A100-80GB")), "h100" | "h100!" => Ok(("H100", "H100!")), "b200" => Ok(("B200", "B200")), - "b200_brev" => Err(anyhow!("Brev profiling requires --profile-brev; --profile uses Modal only")), _ => Err(anyhow!( "GPU '{}' is not supported by local Modal mode. Supported GPUs: T4, L4, L4x4, A100, H100, B200", gpu @@ -220,95 +173,22 @@ fn format_local_result(payload: &Value) -> Result { Ok(sections.join("\n\n")) } -fn save_profile_result(payload: &Value, output_dir: &Path) -> Result { - let result = &payload["result"]; - if result["success"].as_bool() != Some(true) { - return Err(anyhow!("Modal profiling failed: {}", result["error"])); - } - let runs = result["runs"] - .as_object() - .filter(|runs| !runs.is_empty()) - .ok_or_else(|| anyhow!("Modal returned no profile runs"))?; - std::fs::create_dir_all(output_dir)?; - std::fs::write( - output_dir.join("manifest.json"), - serde_json::to_vec_pretty(&serde_json::json!({ - "leaderboard": payload["leaderboard"], - "gpu": payload["gpu"], - "system": result["system"], - "problem_directory": payload["problem_directory"], - "reference_kernels_ref": payload["reference_kernels_ref"], - "kernelbot_ref": payload["kernelbot_ref"], - "profile_options": payload["profile_options"], - "benchmark_specs": payload["benchmark_specs"], - }))?, - )?; - let mut lines = vec![format!( - "Modal Nsight Compute profile: {} on {}", - payload["leaderboard"].as_str().unwrap_or("unknown"), - payload["gpu"].as_str().unwrap_or("unknown") - )]; - let mut failures = Vec::new(); - for (index, (key, run)) in runs.iter().enumerate() { - if let Some(failure) = run_failure(run) { - failures.push(format!("{}: {}", key, failure)); - } - let Some(trace) = run.pointer("/profile/trace").and_then(Value::as_str) else { - failures.push(format!("{}: No NCU report was produced", key)); - continue; - }; - let bytes = base64::engine::general_purpose::STANDARD - .decode(trace) - .context("Modal returned an invalid profile archive")?; - // Use a locally generated filename, never a path from the remote response. - let zip_path = output_dir.join(format!("profile-{}.zip", index)); - std::fs::write(&zip_path, &bytes)?; - let extracted = service::extract_profile_artifacts(&zip_path, &bytes)?; - if extracted.reports.is_empty() { - failures.push(format!("{}: NCU archive contains no .ncu-rep report", key)); - } - lines.push(format!( - "{}: {}", - key, - run.pointer("/run/result/benchmark.0.spec") - .and_then(Value::as_str) - .unwrap_or("unknown benchmark") - )); - for path in extracted.details.iter().chain(extracted.reports.iter()) { - lines.push(format!(" {}", path.display())); - } - } - lines.push(format!( - "Manifest: {}", - output_dir.join("manifest.json").display() - )); - if !failures.is_empty() { - return Err(anyhow!("{}\n\n{}", lines.join("\n"), failures.join("\n"))); - } - Ok(lines.join("\n")) -} - pub async fn run_modal_submission( submission_path: &Path, leaderboard: &str, gpu: &str, mode: &str, - profile_options: &ProfileOptions, ) -> Result { if !matches!( mode.to_ascii_lowercase().as_str(), - "test" | "benchmark" | "leaderboard" | "profile" + "test" | "benchmark" | "leaderboard" ) { return Err(anyhow!( - "Local Modal mode supports test, benchmark, leaderboard, and profile; got '{}'", + "Local Modal mode supports test, benchmark, and leaderboard; got '{}'", mode )); } - profile_options.validate(Some(mode))?; let (kernelbot_gpu, modal_gpu) = gpu_names(gpu)?; - if mode.eq_ignore_ascii_case("profile") && modal_gpu.contains(':') { - return Err(anyhow!("Nsight Compute profiling requires a single GPU")); - } let helper = tempfile::Builder::new() .prefix("popcorn-local-modal-") .suffix(".py") @@ -328,8 +208,6 @@ pub async fn run_modal_submission( .env("POPCORN_LOCAL_GPU", kernelbot_gpu) .env("POPCORN_LOCAL_MODAL_GPU", modal_gpu) .env("POPCORN_LOCAL_MODE", mode.to_ascii_lowercase()) - .env("POPCORN_PROFILE_OPTIONS", serde_json::to_string(profile_options)?) - .kill_on_drop(true) .stdout(Stdio::piped()) .stderr(Stdio::piped()) .spawn() @@ -376,90 +254,13 @@ pub async fn run_modal_submission( } let payload = payload.ok_or_else(|| anyhow!("Modal returned no Popcorn result"))?; let payload: Value = serde_json::from_str(&payload).context("Modal returned invalid JSON")?; - if mode.eq_ignore_ascii_case("profile") { - let output_dir = tempfile::Builder::new() - .prefix("popcorn-profile-") - .tempdir_in(std::env::current_dir()?)? - .into_path(); - save_profile_result(&payload, &output_dir) - } else { - format_local_result(&payload) - } + format_local_result(&payload) } #[cfg(test)] mod tests { use super::*; - fn profile_payload(trace: &str) -> Value { - serde_json::json!({ - "leaderboard": "qr_v2", "gpu": "B200", "result": { - "success": true, "runs": {"profile.3": { - "run": {"success": true, "result": {"benchmark.0.spec": "n: 512"}}, - "profile": {"trace": trace} - }} - } - }) - } - - #[test] - fn extracts_profile_artifacts_and_records_provenance() { - use std::io::{Cursor, Write}; - let mut zip = zip::ZipWriter::new(Cursor::new(Vec::new())); - for name in ["profile.ncu-rep", "ncu-details.txt", "ncu-details.csv"] { - zip.start_file( - format!("profile_data/{}", name), - zip::write::SimpleFileOptions::default(), - ) - .unwrap(); - zip.write_all(b"captured data").unwrap(); - } - let trace = - base64::engine::general_purpose::STANDARD.encode(zip.finish().unwrap().into_inner()); - let mut payload = profile_payload(&trace); - payload["reference_kernels_ref"] = serde_json::json!("verified-sha"); - payload["profile_options"] = serde_json::json!({"benchmark_index": 3}); - let directory = tempfile::tempdir().unwrap(); - let output = save_profile_result(&payload, directory.path()).unwrap(); - assert!(output.contains("profile.3: n: 512")); - assert_eq!( - std::fs::read(directory.path().join("profile-0/profile.ncu-rep")).unwrap(), - b"captured data" - ); - assert!(output.contains("ncu-details.csv")); - let manifest = std::fs::read_to_string(directory.path().join("manifest.json")).unwrap(); - assert!(manifest.contains("verified-sha")); - assert!(manifest.contains("benchmark_index")); - } - - #[test] - fn rejects_missing_reports_and_failed_runs() { - let directory = tempfile::tempdir().unwrap(); - let mut payload = profile_payload(""); - payload["result"]["runs"]["profile.3"]["profile"] = Value::Null; - payload["result"]["runs"]["profile.3"]["run"]["success"] = Value::Bool(false); - payload["result"]["runs"]["profile.3"]["run"]["stderr"] = serde_json::json!("NCU denied"); - let error = save_profile_result(&payload, directory.path()) - .unwrap_err() - .to_string(); - assert!(error.contains("NCU denied")); - assert!(error.contains("No NCU report")); - payload["result"]["runs"] = serde_json::json!({}); - assert!(save_profile_result(&payload, directory.path()).is_err()); - } - - #[test] - fn validates_profile_options_before_launch() { - let mut options = ProfileOptions { - benchmark_index: Some(0), - ..Default::default() - }; - assert!(options.validate(Some("benchmark")).is_err()); - assert!(options.validate(Some("profile")).is_ok()); - options.ncu_launch_count = Some(0); - assert!(options.validate(Some("profile")).is_err()); - } - #[test] fn maps_popcorn_gpu_names_to_modal() { assert_eq!(gpu_names("B200").unwrap(), ("B200", "B200")); diff --git a/src/main.rs b/src/main.rs index 0b36a38..2f2d146 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,7 @@ mod cmd; mod local; mod models; +mod profile; mod service; mod utils; mod views; diff --git a/src/profile.rs b/src/profile.rs new file mode 100644 index 0000000..3616f3b --- /dev/null +++ b/src/profile.rs @@ -0,0 +1,240 @@ +use crate::service; +use anyhow::{anyhow, Context, Result}; +use base64::Engine; +use serde::Serialize; +use serde_json::Value; +use std::path::Path; + +#[derive(clap::Args, Debug, Default, Clone, Serialize)] +pub struct ProfileOptions { + /// Profile benchmarks[N]; omit to profile all benchmark shapes. + #[arg(long)] + pub benchmark_index: Option, + /// Capture only matching NCU kernel names (supports regex: expressions). + #[arg(long)] + pub ncu_kernel_name: Option, + /// How NCU interprets kernel names. + #[arg(long, value_parser = ["function", "demangled", "mangled"])] + pub ncu_kernel_name_base: Option, + /// Maximum matching kernel launches to capture per benchmark (default: 10). + #[arg(long)] + pub ncu_launch_count: Option, +} + +impl ProfileOptions { + pub fn merge(self, fallback: Self) -> Self { + Self { + benchmark_index: self.benchmark_index.or(fallback.benchmark_index), + ncu_kernel_name: self.ncu_kernel_name.or(fallback.ncu_kernel_name), + ncu_kernel_name_base: self.ncu_kernel_name_base.or(fallback.ncu_kernel_name_base), + ncu_launch_count: self.ncu_launch_count.or(fallback.ncu_launch_count), + } + } + + pub fn validate(&self, mode: Option<&str>) -> Result<()> { + if self.ncu_launch_count == Some(0) { + return Err(anyhow!("--ncu-launch-count must be greater than zero")); + } + if (self.benchmark_index.is_some() + || self.ncu_kernel_name.is_some() + || self.ncu_kernel_name_base.is_some() + || self.ncu_launch_count.is_some()) + && !mode.is_some_and(|mode| mode.eq_ignore_ascii_case("profile")) + { + return Err(anyhow!( + "Profiling options require --profile, --profile-brev, or --mode profile" + )); + } + Ok(()) + } +} + +fn run_failure(run: &Value) -> Option { + let compilation = run.get("compilation").filter(|value| !value.is_null()); + if compilation + .and_then(|value| value.get("success")) + .and_then(Value::as_bool) + == Some(false) + { + return Some(format!( + "Compilation failed:\n{}", + compilation + .and_then(|value| value.get("stderr")) + .and_then(Value::as_str) + .unwrap_or("No compiler error was reported") + )); + } + + let result = run.get("run")?; + if result.get("success").and_then(Value::as_bool) == Some(false) { + return Some(format!( + "Execution failed:\n{}", + result + .get("stderr") + .and_then(Value::as_str) + .unwrap_or("No execution error was reported") + )); + } + None +} + +fn save_profile_result(payload: &Value, output_dir: &Path) -> Result { + let result = &payload["result"]; + if result["success"].as_bool() != Some(true) { + return Err(anyhow!("Profiling failed: {}", result["error"])); + } + let runs = result["runs"] + .as_object() + .filter(|runs| !runs.is_empty()) + .ok_or_else(|| anyhow!("Server returned no profile runs"))?; + std::fs::create_dir_all(output_dir)?; + std::fs::write( + output_dir.join("manifest.json"), + serde_json::to_vec_pretty(&serde_json::json!({ + "leaderboard": payload["leaderboard"], + "gpu": payload["gpu"], + "system": result["system"], + "profile_metadata": result["profile_metadata"], + }))?, + )?; + let mut lines = vec![format!( + "Nsight Compute profile: {} on {}", + payload["leaderboard"].as_str().unwrap_or("unknown"), + payload["gpu"].as_str().unwrap_or("unknown") + )]; + let mut failures = Vec::new(); + for (index, (key, run)) in runs.iter().enumerate() { + if let Some(failure) = run_failure(run) { + failures.push(format!("{}: {}", key, failure)); + if let Some(stdout) = run.pointer("/run/stdout").and_then(Value::as_str) { + failures.push(stdout.to_string()); + } + } + let Some(trace) = run.pointer("/profile/trace").and_then(Value::as_str) else { + failures.push(format!("{}: No NCU report was produced", key)); + continue; + }; + let bytes = base64::engine::general_purpose::STANDARD + .decode(trace) + .context("Server returned an invalid profile archive")?; + // Use a locally generated filename, never a path from the remote response. + let zip_path = output_dir.join(format!("profile-{}.zip", index)); + std::fs::write(&zip_path, &bytes)?; + let extracted = service::extract_profile_artifacts(&zip_path, &bytes)?; + if extracted.reports.is_empty() { + failures.push(format!("{}: NCU archive contains no .ncu-rep report", key)); + } + lines.push(format!( + "{}: {}", + key, + run.pointer("/run/result/benchmark.0.spec") + .and_then(Value::as_str) + .unwrap_or("unknown benchmark") + )); + for path in extracted.details.iter().chain(extracted.reports.iter()) { + lines.push(format!(" {}", path.display())); + } + } + lines.push(format!( + "Manifest: {}", + output_dir.join("manifest.json").display() + )); + if !failures.is_empty() { + return Err(anyhow!("{}\n\n{}", lines.join("\n"), failures.join("\n"))); + } + Ok(lines.join("\n")) +} + +pub fn save_hosted_results(response: &Value, leaderboard: &str, gpu: &str) -> Result { + let results = response["results"] + .as_array() + .filter(|results| !results.is_empty()) + .ok_or_else(|| anyhow!("Server returned no profiling results"))?; + let output_dir = tempfile::Builder::new() + .prefix("popcorn-profile-") + .tempdir_in(std::env::current_dir()?)? + .into_path(); + let mut summaries = Vec::new(); + for (index, result) in results.iter().enumerate() { + summaries.push(save_profile_result( + &serde_json::json!({ + "leaderboard": leaderboard, "gpu": gpu, "result": result, + }), + &output_dir.join(format!("result-{}", index)), + )?); + } + Ok(summaries.join("\n\n")) +} + +#[cfg(test)] +mod tests { + use super::*; + fn profile_payload(trace: &str) -> Value { + serde_json::json!({ + "leaderboard": "qr_v2", "gpu": "B200", "result": { + "success": true, "runs": {"profile.3": { + "run": {"success": true, "result": {"benchmark.0.spec": "n: 512"}}, + "profile": {"trace": trace} + }} + } + }) + } + + #[test] + fn extracts_profile_artifacts_and_records_provenance() { + use std::io::{Cursor, Write}; + let mut zip = zip::ZipWriter::new(Cursor::new(Vec::new())); + for name in ["profile.ncu-rep", "ncu-details.txt", "ncu-details.csv"] { + zip.start_file( + format!("profile_data/{}", name), + zip::write::SimpleFileOptions::default(), + ) + .unwrap(); + zip.write_all(b"captured data").unwrap(); + } + let trace = + base64::engine::general_purpose::STANDARD.encode(zip.finish().unwrap().into_inner()); + let mut payload = profile_payload(&trace); + payload["result"]["profile_metadata"] = serde_json::json!({"config_sha256": "verified-sha", "capture_options": {"benchmark_index": 3}}); + payload["profile_options"] = serde_json::json!({"benchmark_index": 3}); + let directory = tempfile::tempdir().unwrap(); + let output = save_profile_result(&payload, directory.path()).unwrap(); + assert!(output.contains("profile.3: n: 512")); + assert_eq!( + std::fs::read(directory.path().join("profile-0/profile.ncu-rep")).unwrap(), + b"captured data" + ); + assert!(output.contains("ncu-details.csv")); + let manifest = std::fs::read_to_string(directory.path().join("manifest.json")).unwrap(); + assert!(manifest.contains("verified-sha")); + assert!(manifest.contains("benchmark_index")); + } + + #[test] + fn rejects_missing_reports_and_failed_runs() { + let directory = tempfile::tempdir().unwrap(); + let mut payload = profile_payload(""); + payload["result"]["runs"]["profile.3"]["profile"] = Value::Null; + payload["result"]["runs"]["profile.3"]["run"]["success"] = Value::Bool(false); + payload["result"]["runs"]["profile.3"]["run"]["stderr"] = serde_json::json!("NCU denied"); + let error = save_profile_result(&payload, directory.path()) + .unwrap_err() + .to_string(); + assert!(error.contains("NCU denied")); + assert!(error.contains("No NCU report")); + payload["result"]["runs"] = serde_json::json!({}); + assert!(save_profile_result(&payload, directory.path()).is_err()); + } + + #[test] + fn validates_profile_options_before_launch() { + let mut options = ProfileOptions { + benchmark_index: Some(0), + ..Default::default() + }; + assert!(options.validate(Some("benchmark")).is_err()); + assert!(options.validate(Some("profile")).is_ok()); + options.ncu_launch_count = Some(0); + assert!(options.validate(Some("profile")).is_err()); + } +} diff --git a/src/service/mod.rs b/src/service/mod.rs index b72c71d..96676bf 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -629,6 +629,7 @@ pub async fn submit_solution>( leaderboard, gpu, submission_mode, + None, on_log, ) .await; @@ -646,12 +647,35 @@ pub async fn submit_solution>( .await } +#[allow(clippy::too_many_arguments)] +pub async fn profile_solution>( + client: &Client, + filepath: P, + file_content: &[u8], + leaderboard: &str, + gpu: &str, + options: &crate::profile::ProfileOptions, + on_log: Option>, +) -> Result { + submit_solution_streaming( + client, + filepath, + file_content, + leaderboard, + gpu, + "profile", + Some(options), + on_log, + ) + .await +} + pub async fn profile_brev_solution>( client: &Client, filepath: P, file_content: &[u8], leaderboard: &str, - options: &crate::local::ProfileOptions, + options: &crate::profile::ProfileOptions, on_log: Option>, ) -> Result { let base_url = env::var("POPCORN_BREV_PROFILER_URL") @@ -1611,6 +1635,7 @@ fn format_submission_details(details: &SubmissionDetails) -> Result { } } +#[allow(clippy::too_many_arguments)] async fn submit_solution_streaming>( client: &Client, filepath: P, @@ -1618,6 +1643,7 @@ async fn submit_solution_streaming>( leaderboard: &str, gpu: &str, submission_mode: &str, + profile_options: Option<&crate::profile::ProfileOptions>, on_log: Option>, ) -> Result { let base_url = @@ -1631,15 +1657,38 @@ async fn submit_solution_streaming>( let part = Part::bytes(file_content.to_vec()).file_name(filename.to_string()); - let form = Form::new().part("file", part); + let mut form = Form::new().part("file", part); + if let Some(options) = profile_options { + options.validate(Some("profile"))?; + for (key, value) in serde_json::to_value(options)?.as_object().unwrap() { + if !value.is_null() { + form = form.text( + key.clone(), + value + .as_str() + .map(str::to_owned) + .unwrap_or_else(|| value.to_string()), + ); + } + } + } - let url = format!( - "{}/{}/{}/{}", - base_url, - leaderboard.to_lowercase(), - gpu, - submission_mode.to_lowercase() - ); + let url = if profile_options.is_some() { + format!( + "{}/profile/{}/{}", + base_url, + leaderboard.to_lowercase(), + gpu + ) + } else { + format!( + "{}/{}/{}/{}", + base_url, + leaderboard.to_lowercase(), + gpu, + submission_mode.to_lowercase() + ) + }; let resp = client .post(&url) @@ -1706,6 +1755,13 @@ async fn submit_solution_streaming>( } "result" => { let result_val: Value = serde_json::from_str(data)?; + if profile_options.is_some() { + return crate::profile::save_hosted_results( + &result_val, + leaderboard, + gpu, + ); + } if let Some(ref cb) = on_log { // Handle "results" array @@ -1816,6 +1872,9 @@ async fn submit_solution_streaming>( )) } else { let result: Value = resp.json().await?; + if profile_options.is_some() { + return crate::profile::save_hosted_results(&result, leaderboard, gpu); + } let pretty_result = match result.get("results") { Some(result_obj) => serde_json::to_string_pretty(result_obj)?, None => return Err(anyhow!("Invalid non-streaming response structure")), @@ -2097,6 +2156,56 @@ mod tests { String::from_utf8(request).unwrap() } + #[tokio::test] + async fn hosted_profile_uses_popcorn_auth_and_request_options() { + let _env_guard = ENV_LOCK.lock().await; + let original = std::env::var("POPCORN_API_URL").ok(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + std::env::set_var( + "POPCORN_API_URL", + format!("http://{}", listener.local_addr().unwrap()), + ); + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + let request = read_http_request(&mut stream).await; + assert!(request.starts_with("POST /profile/qr_v2/B200 HTTP/1.1")); + assert!(request + .to_lowercase() + .contains("x-popcorn-cli-id: test-user")); + for field in ["benchmark_index", "ncu_kernel_name", "ncu_launch_count"] { + assert!(request.contains(&format!("name=\"{}\"", field))); + } + assert!(request.contains("regex:solver")); + let body = "event: error\ndata: {\"detail\":\"deliberate capture failure\"}\n\n"; + stream.write_all(format!("HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", body.len(), body).as_bytes()).await.unwrap(); + }); + let client = create_client(Some("test-user".to_string())).unwrap(); + let result = profile_solution( + &client, + "submission.py", + b"pass", + "qr_v2", + "B200", + &crate::profile::ProfileOptions { + benchmark_index: Some(1), + ncu_kernel_name: Some("regex:solver".to_string()), + ncu_launch_count: Some(2), + ..Default::default() + }, + None, + ) + .await; + server.await.unwrap(); + match original { + Some(value) => std::env::set_var("POPCORN_API_URL", value), + None => std::env::remove_var("POPCORN_API_URL"), + } + assert!(result + .unwrap_err() + .to_string() + .contains("deliberate capture failure")); + } + #[tokio::test] async fn test_standard_submission_modes_use_background_results_except_profile() { let _env_guard = ENV_LOCK.lock().await; diff --git a/templates/local_modal_runner.py b/templates/local_modal_runner.py index 81f1945..6157d14 100644 --- a/templates/local_modal_runner.py +++ b/templates/local_modal_runner.py @@ -5,9 +5,6 @@ """ import dataclasses -import base64 -import functools -import subprocess import glob import json import os @@ -136,20 +133,10 @@ def _github_ref(repo: str, override_env: str) -> str: "/opt/cutlass/include:/opt/cutlass/tools/util/include" ), "PYTHONPATH": "/opt/kernelbot/src", - # The remote worker imports this module too. Carry only public build - # configuration across, never the user's local paths or credentials. - "POPCORN_REFERENCE_KERNELS_REF": reference_ref, - "POPCORN_KERNELBOT_REF": kernelbot_ref, - "POPCORN_LOCAL_MODAL_GPU": os.environ["POPCORN_LOCAL_MODAL_GPU"], - "POPCORN_LOCAL_MODE": os.environ["POPCORN_LOCAL_MODE"], } ) ) -# CUDA devel includes NCU. Verify the executable at image build time. -if os.environ.get("POPCORN_LOCAL_MODE") == "profile": - cuda_image = cuda_image.run_commands("ncu --version") - app = modal.App("popcorn-local-runner", image=cuda_image) modal_gpu = os.environ["POPCORN_LOCAL_MODAL_GPU"] @@ -179,76 +166,11 @@ def _find_problem(leaderboard: str) -> tuple[Path, list[str]]: return Path("/opt/reference-kernels/problems") / directory / "task.yml", supported_gpus -def _select_benchmarks(config: dict, options: dict) -> dict: - benchmarks = config.get("benchmarks", []) - if not benchmarks: - raise ValueError("This task has no benchmark shapes to profile") - index = options.get("benchmark_index") - if index is None: - return config - if not isinstance(index, int) or index < 0 or index >= len(benchmarks): - raise ValueError(f"Benchmark index {index} is out of range (0..{len(benchmarks) - 1})") - return {**config, "benchmarks": [benchmarks[index]]} - - -def _ncu_command(call: list[str], output_dir: Path, options: dict) -> list[str]: - count = options.get("ncu_launch_count") - if count is None: - count = 10 - if not isinstance(count, int) or count <= 0: - raise ValueError("NCU launch count must be a positive integer") - command = [ - "ncu", "--set", "full", "--target-processes", "all", - "--nvtx", "--nvtx-include", "custom_kernel/", - "--import-source", "1", "--launch-count", str(count), - "--cache-control", "all", "--clock-control", "none", - "--replay-mode", "kernel", "--force-overwrite", - "--export", str(output_dir / "profile.ncu-rep"), - ] - for key, flag in [("ncu_kernel_name", "--kernel-name"), - ("ncu_kernel_name_base", "--kernel-name-base")]: - if options.get(key): - command.extend([flag, options[key]]) - return command + ["--", *call] - - -def _profile_ncu(call, seed, timeout, multi_gpu, output_dir, *, options): - from libkernelbot.run_eval import ProfileResult, _directory_to_zip_bytes, run_program - - if multi_gpu: - raise ValueError("Nsight Compute profiling requires a single GPU") - result = run_program( - _ncu_command(call, output_dir, options), seed=seed, timeout=timeout, - multi_gpu=False, extra_env={"POPCORN_NCU": "1"}, - ) - report = output_dir / "profile.ncu-rep" - if not result.success or not report.is_file(): - result.success = False - result.stderr += "\nNsight Compute did not produce a report. Check NCU output and the evaluator's profile mode/NVTX range." - return result, None - for name, extra in [("ncu-details.txt", []), ("ncu-details.csv", ["--csv"])]: - details = subprocess.run( - ["ncu", "--import", str(report), "--page", "details", *extra], - stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=120, - ) - if details.returncode: - result.success = False - result.stderr += f"\nFailed to export {name}: {details.stderr}" - else: - (output_dir / name).write_text(details.stdout) - details_path = output_dir / "ncu-details.txt" - if details_path.exists(): - result.result["benchmark.0.report"] = base64.b64encode(details_path.read_bytes()).decode() - return result, ProfileResult( - profiler="Nsight-Compute", trace=_directory_to_zip_bytes(output_dir), download_url=None, - ) - - @app.function(gpu=modal_gpu, timeout=3600) -def evaluate(submission: str, leaderboard: str, gpu: str, mode: str, profile_options: dict) -> dict: +def evaluate(submission: str, leaderboard: str, gpu: str, mode: str) -> dict: try: from libkernelbot.consts import GPU_TO_SM, SubmissionMode - from libkernelbot import run_eval + from libkernelbot.run_eval import run_config from libkernelbot.task import build_task_config, make_task_definition task_path, supported_gpus = _find_problem(leaderboard) @@ -265,29 +187,13 @@ def evaluate(submission: str, leaderboard: str, gpu: str, mode: str, profile_opt arch=GPU_TO_SM[gpu], mode=SubmissionMode(mode), ) - if mode == "profile": - config = _select_benchmarks(config, profile_options) - # Keep KernelBot's task setup, timeouts and per-shape evaluator execution. - # Only replace NCU capture to add Modal-safe clocks and CLI filters. - original_profile = run_eval.profile_program_ncu - try: - run_eval.profile_program_ncu = functools.partial(_profile_ncu, options=profile_options) - result = run_eval.run_config(config) - finally: - run_eval.profile_program_ncu = original_profile - index = profile_options.get("benchmark_index") - if index is not None and "profile.0" in result.runs: - result.runs[f"profile.{index}"] = result.runs.pop("profile.0") - else: - result = run_eval.run_config(config) + result = run_config(config) return { "leaderboard": leaderboard, "problem_directory": str(task_path.parent.relative_to("/opt/reference-kernels/problems")), "gpu": gpu, "mode": mode, "ranking_by": definition.task.ranking_by.value, - "profile_options": profile_options, - "benchmark_specs": config.get("benchmarks", []), "reference_kernels_ref": reference_ref, "kernelbot_ref": kernelbot_ref, "result": dataclasses.asdict(result), @@ -316,6 +222,5 @@ def main(): os.environ["POPCORN_LOCAL_LEADERBOARD"], os.environ["POPCORN_LOCAL_GPU"], os.environ["POPCORN_LOCAL_MODE"], - json.loads(os.environ.get("POPCORN_PROFILE_OPTIONS", "{}")), ) print(RESULT_MARKER + json.dumps(payload, default=str, separators=(",", ":"))) diff --git a/tests/test_modal_profile.py b/tests/test_modal_profile.py deleted file mode 100644 index e213322..0000000 --- a/tests/test_modal_profile.py +++ /dev/null @@ -1,46 +0,0 @@ -"""CPU checks for the embedded runner; importing the full module builds a Modal image.""" -import ast -import unittest -from pathlib import Path - -RUNNER = Path(__file__).resolve().parents[1] / "templates/local_modal_runner.py" -namespace = {"Path": Path} -tree = ast.parse(RUNNER.read_text()) -helpers = ast.Module(body=[node for node in tree.body if isinstance(node, ast.FunctionDef) - and node.name in {"_ncu_command", "_select_benchmarks"}], type_ignores=[]) -exec(compile(helpers, str(RUNNER), "exec"), namespace) - - -class ModalProfileTests(unittest.TestCase): - def test_selected_shape_and_all_shapes(self): - config = {"benchmarks": [{"n": 32}, {"n": 512}], "tests": [{"n": 16}]} - select = namespace["_select_benchmarks"] - self.assertIs(select(config, {}), config) - self.assertEqual(select(config, {"benchmark_index": 1})["benchmarks"], [{"n": 512}]) - self.assertEqual(len(config["benchmarks"]), 2) - self.assertEqual(select(config, {"benchmark_index": 1})["tests"], [{"n": 16}]) - for index in [-1, 2]: - with self.assertRaises(ValueError): - select(config, {"benchmark_index": index}) - with self.assertRaises(ValueError): - select({"benchmarks": []}, {}) - - def test_capture_tracks_child_processes_without_clock_control(self): - command = namespace["_ncu_command"](["python3", "eval.py"], Path("/tmp/out"), {}) - self.assertEqual(command[-3:], ["--", "python3", "eval.py"]) - for flag, value in [("--clock-control", "none"), ("--target-processes", "all"), - ("--nvtx-include", "custom_kernel/"), ("--launch-count", "10")]: - self.assertEqual(command[command.index(flag) + 1], value) - - def test_filters_remain_single_arguments(self): - command = namespace["_ncu_command"](["program"], Path("/tmp/out"), { - "ncu_kernel_name": "regex:late_kernel|kernel with spaces", - "ncu_kernel_name_base": "demangled", "ncu_launch_count": 2, - }) - self.assertEqual(command[command.index("--kernel-name") + 1], "regex:late_kernel|kernel with spaces") - self.assertEqual(command[command.index("--kernel-name-base") + 1], "demangled") - self.assertEqual(command[command.index("--launch-count") + 1], "2") - - -if __name__ == "__main__": - unittest.main()