diff --git a/.gitignore b/.gitignore index 47e7497..1893e8a 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,7 @@ scratch.md .codex/ .popcorn/ .DS_Store + +popcorn-profile-*/ +validation/ +__pycache__/ diff --git a/README.md b/README.md index c0f6c08..ab5042a 100644 --- a/README.md +++ b/README.md @@ -7,39 +7,38 @@ Tested on linux and mac but should just work on Windows as well. ## New: Nsight Compute Profiling -Profile submissions on the hosted GPU Mode B200 Nsight Compute service. +Profile submissions through GPU Mode (B200 by default). +Use your normal Popcorn registration. `--profile` and `--mode profile` submit +through GPU Mode and save NCU reports locally; no provider account or SDK is +needed. Use `--profile-brev` to explicitly select the Brev service. See [docs/profiling.md](docs/profiling.md) for a complete copy-paste flow. Quick QR v2 example: ```bash curl -O https://raw.githubusercontent.com/gpu-mode/reference-kernels/main/problems/linalg/qr_v2/submission.py -export POPCORN_BREV_PROFILER_URL=https://http--brev-profiler-proxy--dxfjds728w5v.code.run -popcorn submit submission.py --leaderboard qr_v2 --profile-brev --benchmark-index 0 --no-tui +popcorn submit submission.py --leaderboard qr_v2 --profile --benchmark-index 0 --no-tui ``` Quick `eigh` dense row example: ```bash curl -O https://raw.githubusercontent.com/gpu-mode/reference-kernels/main/problems/linalg/eigh_py/submission.py -export POPCORN_BREV_PROFILER_URL=https://http--brev-profiler-proxy--dxfjds728w5v.code.run -popcorn submit submission.py --leaderboard eigh --profile-brev --benchmark-index 3 --no-tui +popcorn submit submission.py --leaderboard eigh --profile --benchmark-index 3 --no-tui ``` Quick `cholesky` example: ```bash curl -O https://raw.githubusercontent.com/gpu-mode/reference-kernels/main/problems/linalg/cholesky_py/submission.py -export POPCORN_BREV_PROFILER_URL=https://http--brev-profiler-proxy--dxfjds728w5v.code.run -popcorn submit submission.py --leaderboard cholesky --profile-brev --benchmark-index 0 --no-tui +popcorn submit submission.py --leaderboard cholesky --profile --benchmark-index 0 --no-tui ``` The CLI downloads and extracts `ncu-details.txt` and `ncu-details.csv` for -agent-readable analysis. It also extracts the optional `.ncu-rep` GUI report and -ends with a macOS command that opens it in Nsight Compute: +agent-readable analysis. It also extracts the optional `.ncu-rep` GUI report which you can open in Nsight Compute: ```bash -open -a "NVIDIA Nsight Compute" profile.0-.../profile.ncu-rep +open -a "NVIDIA Nsight Compute" popcorn-profile-/profile-0/profile.ncu-rep ``` ## [NEW] Submit To The Linear Algebra Competition @@ -139,14 +138,14 @@ popcorn submit solution.py # Direct submission with all options popcorn submit --leaderboard grayscale_v2 --gpu A100 --mode leaderboard solution.py -# Nsight Compute profile on the hosted GPU Mode B200 profiler -POPCORN_BREV_PROFILER_URL=https://http--brev-profiler-proxy--dxfjds728w5v.code.run popcorn submit --leaderboard qr_v2 --profile-brev solution.py +# Nsight Compute profile through GPU Mode (B200 by default) +popcorn submit --leaderboard qr_v2 --profile solution.py # Profile one QR v2 benchmark shape -POPCORN_BREV_PROFILER_URL=https://http--brev-profiler-proxy--dxfjds728w5v.code.run popcorn submit --leaderboard qr_v2 --profile-brev --benchmark-index 0 solution.py +popcorn submit --leaderboard qr_v2 --profile --benchmark-index 0 solution.py # Profile one eigh benchmark shape -POPCORN_BREV_PROFILER_URL=https://http--brev-profiler-proxy--dxfjds728w5v.code.run popcorn submit --leaderboard eigh --profile-brev --benchmark-index 3 solution.py +popcorn submit --leaderboard eigh --profile --benchmark-index 3 solution.py # Plain output mode (no TUI, good for CI/scripts) popcorn submit --no-tui --leaderboard grayscale_v2 --gpu A100 --mode test solution.py @@ -200,7 +199,7 @@ if either lookup fails, it stops instead of risking a stale cached image. - `test` - Quick test run to check correctness - `benchmark` - Benchmark your solution (no leaderboard impact) - `leaderboard` - Official ranked submission -- `profile` - Profile with Nsight Compute (limited availability) +- `profile` - Profile with Nsight Compute through GPU Mode (default GPU: B200) ### Submissions diff --git a/docs/linalg-qr-b200.md b/docs/linalg-qr-b200.md index 47c7a1e..eb519b1 100644 --- a/docs/linalg-qr-b200.md +++ b/docs/linalg-qr-b200.md @@ -22,8 +22,8 @@ popcorn submit --leaderboard qr_v2 --gpu B200 --mode test submission.py Profile the first benchmark shape with Nsight Compute: ```bash -export POPCORN_BREV_PROFILER_URL=https://http--brev-profiler-proxy--dxfjds728w5v.code.run -popcorn submit --leaderboard qr_v2 --profile-brev --benchmark-index 0 submission.py +popcorn register discord +popcorn submit --leaderboard qr_v2 --profile --benchmark-index 0 submission.py ``` The CLI downloads a `.zip` and extracts `ncu-details.txt`, `ncu-details.csv`, diff --git a/docs/modal-ncu-validation.md b/docs/modal-ncu-validation.md new file mode 100644 index 0000000..a34b745 --- /dev/null +++ b/docs/modal-ncu-validation.md @@ -0,0 +1,45 @@ +# Hosted NCU profiling validation + +`--profile` uses Popcorn authentication and the KernelBot API. KernelBot owns +Modal credentials and executes NCU; the client only uploads code and saves +returned artifacts. `--profile-brev` remains explicit. A capture error never +switches providers. + +The CLI calls `POST /profile/{leaderboard}/{gpu}` with multipart capture options. +This new endpoint requires the companion KernelBot update. Deploy the API and +GPU runner before releasing the CLI; older services do not support this route. +The existing `--local` evaluation workflow is independent of hosted profiling. + +Validation covers: + +- Rust tests for authenticated HTTP submission, multipart options, SSE errors, + artifact extraction, and malformed/missing captures. +- KernelBot tests for authenticated API requests, option validation, benchmark + selection, child-process capture, and report exports. +- An isolated integration run exercised the real FastAPI route, submission + preparation, KernelBackend, GPU `run_config`, and CLI artifact extraction, + with a fake database and an ephemeral B200 launcher. The CLI used normal + Popcorn header authentication with an empty `PATH` and no `MODAL_*` values. + It returned a 7,593,257-byte report plus text/CSV exports. +- NCU 2025.2.1 captured 39 passes. The exported report contained 322.78 us + duration, 12.51% achieved occupancy, 0.42% SM throughput, 70.09% L2 hit rate, + and 1,565,428 executed instructions. Six `ctc__*` metrics were unavailable. +- NCU 2026.2.0 from CUDA 13.3 produced many NaN counters on this Modal B200. + The server image therefore pins the tested 2025.2.1 version. The earlier + 2026.2 report was evidence of artifact delivery, not healthy counter coverage. + +GPU fixture provenance: `problems/linalg/qr_v2`, benchmark 0 +(`batch=20, n=32, cond=1, seed=43214`), reference-kernels +`51e22db671d36c1c76091c43c36a44546ba324a1` (the subsequent guide commit changes +only documentation). The initial KernelBot baseline was +`30ba5ce79107e5405b0cc1eda48ca551e7a51b16`. + +Only QR v2 has been tested end to end for this change. Other problems need an +NCU-compatible evaluator; accepting `profile` with only a PyTorch profiler path +is insufficient. Single-GPU NVIDIA capture is supported. Some requested metrics +can be unavailable on a particular GPU. + +The tested image used CUDA 13.3 and PyTorch 2.12.0+cu130, with +`regex:geqr2`, demangled kernel names, and launch count 1. The isolated test +is not a production deployment. [Operator validation run](https://modal.com/apps/coreauto/main/ap-p8eodYfHkZAMp0ZT5btR1y) +requires access to the operator's workspace. diff --git a/docs/profiling.md b/docs/profiling.md index 1bc3e3c..b402e36 100644 --- a/docs/profiling.md +++ b/docs/profiling.md @@ -1,30 +1,51 @@ # Nsight Compute Profiling -This profiles GPU Mode submissions on the hosted B200 Nsight Compute service and +The default profiler submits through GPU Mode on B200 and downloads agent-readable `ncu-details.txt` / `ncu-details.csv` artifacts. The full `.ncu-rep` GUI report is still included for local inspection. -The profiler uses the `benchmarks:` list from the active `reference-kernels` -checkout. `--benchmark-index N` profiles `benchmarks[N]`; omitting +The profiler uses the `benchmarks:` list from the task synced into the hosted +leaderboard configuration. `--benchmark-index N` profiles `benchmarks[N]`; omitting `--benchmark-index` profiles every benchmark entry for that leaderboard. -## 1. Install and Register +## Supported problems + +The task evaluator must implement `profile` mode and launch the submission in +an NVTX push/pop range named `custom_kernel`. A PyTorch-profiler-only path is +not sufficient. QR, QR v2, Eigh, Cholesky, and the shared NVIDIA evaluator have +the required NCU path; only QR v2 has been tested end to end for this CLI change. +Check the actual evaluator selected by the task, since task-specific copies may +have different support. AMD and multi-GPU NCU profiling are unsupported. + +Problem authors can follow the reference-kernels +[NCU integration guide](https://github.com/gpu-mode/reference-kernels/blob/a8044f1658acd4104558bedd3e78a8f096fd778a/docs/ncu-profiling.md). + +## 1. Install and register ```bash curl -fsSL https://raw.githubusercontent.com/gpu-mode/popcorn-cli/main/install.sh | bash popcorn register discord ``` -Restart your terminal if `popcorn` is not found after installation. +`--profile` and `--mode profile` use the normal authenticated GPU Mode API and +imply plain output. No Modal installation, provider account, provider token, +or profiling URL is needed. The service owns the compute credentials. B200 is +the default GPU; use `--gpu` or a submission GPU directive to select another +supported GPU. The existing `--local` evaluation option is a separate workflow +for users deliberately choosing their own compute account. -## 2. Set the Hosted Profiler URL +## 2. Benchmark selection and artifacts -```bash -export POPCORN_BREV_PROFILER_URL=https://http--brev-profiler-proxy--dxfjds728w5v.code.run -``` +The hosted service uses the task/evaluator already synced to that leaderboard. +Local reference-kernels edits and `POPCORN_REFERENCE_KERNELS_REF` do not change +a hosted run. Problem authors should ask an operator to sync their published +revision before validating it. -`BREV_PROFILER_URL` is also accepted as a fallback, but -`POPCORN_BREV_PROFILER_URL` is preferred. +Each profile saves a `manifest.json` containing the leaderboard, GPU/system +information, selected benchmark specs, capture options, and an evaluation-config +SHA-256 digest. Reports go into a unique `popcorn-profile-*` directory. +`--output` saves the text summary; artifacts remain in that directory. The +config digest identifies the evaluated content; it is not a repository commit. ## 3. Profile QR v2 @@ -41,7 +62,7 @@ Profile one benchmark shape: ```bash popcorn submit submission.py \ --leaderboard qr_v2 \ - --profile-brev \ + --profile \ --benchmark-index 0 \ --no-tui ``` @@ -67,14 +88,13 @@ Profile the dense `n=512` leverage row: ```bash popcorn submit submission.py \ --leaderboard eigh \ - --profile-brev \ + --profile \ --benchmark-index 3 \ --no-tui ``` -The hosted profiler uses a deeper Nsight Compute launch window for `eigh` than -for QR v2 so PyTorch/cuSOLVER submissions can reach solver-path kernels after -clone/setup launches. +The default capture window is 10 kernel launches per benchmark. For late solver +kernels, select a kernel name or increase `--ncu-launch-count`. Current `eigh` benchmark index table from `reference-kernels` main `4a1153e`: @@ -110,31 +130,41 @@ Profile the `batch=4096, n=32` benchmark: ```bash popcorn submit submission.py \ --leaderboard cholesky \ - --profile-brev \ + --profile \ --benchmark-index 0 \ --no-tui ``` -The profiler will only profile the first 10 kernels, so if your solution uses more than that (the default Pytorch implementation does), then the profiler might be of limited use. +Capture late kernels with the same filters on any leaderboard: + +```bash +popcorn submit submission.py --leaderboard cholesky --profile --benchmark-index 0 \ + --ncu-kernel-name 'regex:my_kernel' --ncu-kernel-name-base demangled \ + --ncu-launch-count 2 +``` + +NCU follows child processes, captures the evaluator's `custom_kernel` NVTX range, +and leaves GPU clocks unchanged. Empty or failed captures return an error. +Multi-GPU profiling is unsupported. ## 6. Read the Details After the run finishes, the CLI downloads and extracts files like: ```text -profile.0-batch-20-n-32-cond-1-seed-43214.zip -profile.0-batch-20-n-32-cond-1-seed-43214/ncu-details.txt -profile.0-batch-20-n-32-cond-1-seed-43214/ncu-details.csv -profile.0-batch-20-n-32-cond-1-seed-43214/profile.ncu-rep # optional GUI report +popcorn-profile-/result-0/profile-0.zip +popcorn-profile-/result-0/profile-0/ncu-details.txt +popcorn-profile-/result-0/profile-0/ncu-details.csv +popcorn-profile-/result-0/profile-0/profile.ncu-rep # optional GUI report ``` Use `ncu-details.txt` or `ncu-details.csv` as the default artifact for AI -analysis. The CLI prints clickable links for these detail files. +analysis. The CLI prints local paths for the detail files and report. -The last line printed by the CLI opens the optional GUI report on macOS: +Open the GUI report on macOS: ```bash -open -a "NVIDIA Nsight Compute" 'profile.0-batch-20-n-32-cond-1-seed-43214/profile.ncu-rep' +open -a "NVIDIA Nsight Compute" 'popcorn-profile-/result-0/profile-0/profile.ncu-rep' ``` ## Profile All Benchmark Shapes @@ -144,7 +174,7 @@ Omit `--benchmark-index`: ```bash popcorn submit submission.py \ --leaderboard eigh \ - --profile-brev \ + --profile \ --no-tui ``` @@ -165,3 +195,18 @@ For leaderboard submission: ```bash popcorn submit submission.py --leaderboard qr_v2 --gpu B200 --mode leaderboard --no-tui ``` + +## Explicit Brev profiling + +Use `--profile-brev` to select the hosted Brev service explicitly. `--profile` +uses the GPU Mode API; errors never trigger a switch to Brev. Brev requires Popcorn +registration: + +```bash +popcorn register discord +export POPCORN_BREV_PROFILER_URL=https://http--brev-profiler-proxy--dxfjds728w5v.code.run +popcorn submit submission.py --leaderboard qr_v2 --profile-brev --benchmark-index 0 +``` + +`BREV_PROFILER_URL` is also accepted. Brev profiling uses that service's deployed +reference-kernels checkout. diff --git a/src/cmd/mod.rs b/src/cmd/mod.rs index 39dfeb2..63ad695 100644 --- a/src/cmd/mod.rs +++ b/src/cmd/mod.rs @@ -62,6 +62,10 @@ pub struct Cli { #[arg(long)] pub mode: Option, + /// Profile with Nsight Compute through GPU Mode (default GPU: B200). + #[arg(long, conflicts_with = "profile_brev")] + pub profile: bool, + /// Profile on the hosted GPU Mode Brev B200 and save Nsight Compute artifacts locally. /// Requires POPCORN_BREV_PROFILER_URL, or BREV_PROFILER_URL as a fallback. #[arg(long)] @@ -72,9 +76,8 @@ pub struct Cli { #[arg(long, conflicts_with = "profile_brev")] pub local: bool, - /// Optional: Profile a single benchmark index when using --profile-brev - #[arg(long)] - pub benchmark_index: Option, + #[command(flatten)] + pub profile_options: crate::profile::ProfileOptions, // Optional: Specify output file #[arg(short, long)] @@ -151,6 +154,10 @@ enum Commands { #[arg(long)] mode: Option, + /// Profile with Nsight Compute through GPU Mode (default GPU: B200). + #[arg(long, conflicts_with = "profile_brev")] + profile: bool, + /// Profile on the hosted GPU Mode Brev B200 and save Nsight Compute artifacts locally. /// Requires POPCORN_BREV_PROFILER_URL, or BREV_PROFILER_URL as a fallback. #[arg(long)] @@ -161,9 +168,8 @@ enum Commands { #[arg(long, conflicts_with = "profile_brev")] local: bool, - /// Optional: Profile a single benchmark index when using --profile-brev - #[arg(long)] - benchmark_index: Option, + #[command(flatten)] + profile_options: crate::profile::ProfileOptions, // Optional: Specify output file #[arg(short, long)] @@ -213,26 +219,47 @@ pub async fn execute(cli: Cli) -> Result<()> { leaderboard, mode, profile_brev, + profile, local, - benchmark_index, + profile_options, output, no_tui, }) => { // Use filepath from Submit command first, fallback to top-level filepath let final_filepath = filepath.or(cli.filepath); + let profile_brev = profile_brev || cli.profile_brev; + let profile = profile || cli.profile; + let local = local || cli.local; + if (profile_brev && (profile || local)) || (local && profile) { + return Err(anyhow!( + "Profiling uses the hosted service; --local cannot be combined with profiling, and --profile conflicts with --profile-brev" + )); + } + let profile_options = profile_options.merge(cli.profile_options); let final_gpu = if profile_brev { Some("B200_Brev".to_string()) } else { - gpu.clone() + gpu.clone().or(cli.gpu.clone()) }; - let final_mode = if profile_brev { + let final_mode = if profile_brev || profile { Some("profile".to_string()) } else { - mode.clone() + mode.clone().or(cli.mode.clone()) }; + profile_options.validate(final_mode.as_deref())?; + if local && is_profile_mode(final_mode.as_deref()) { + return Err(anyhow!("Profiling uses the hosted service; omit --local")); + } if local { - submit::run_submit_local(final_filepath, gpu, leaderboard, mode, output).await + submit::run_submit_local( + final_filepath, + final_gpu, + leaderboard.or(cli.leaderboard), + final_mode, + output, + ) + .await } else { let config = load_config()?; let cli_id = config.cli_id.ok_or_else(|| { @@ -245,14 +272,15 @@ pub async fn execute(cli: Cli) -> Result<()> { ) })?; - if no_tui || profile_brev { + if no_tui || cli.no_tui || is_profile_mode(final_mode.as_deref()) { submit::run_submit_plain( final_filepath, // Resolved filepath final_gpu, // From Submit command - leaderboard, // From Submit command - final_mode, // From Submit command + leaderboard.or(cli.leaderboard), + final_mode, // From Submit command cli_id, - benchmark_index.or(cli.benchmark_index), + profile_options, + profile_brev, output, // From Submit command ) .await @@ -260,8 +288,8 @@ pub async fn execute(cli: Cli) -> Result<()> { submit::run_submit_tui( final_filepath, // Resolved filepath final_gpu, // From Submit command - leaderboard, // From Submit command - final_mode, // From Submit command + leaderboard.or(cli.leaderboard), + final_mode, // From Submit command cli_id, output, // From Submit command ) @@ -318,6 +346,8 @@ pub async fn execute(cli: Cli) -> Result<()> { None => { // Check if any of the submission-related flags were used at the top level if !cli.profile_brev + && !cli.profile + && !is_profile_mode(cli.mode.as_deref()) && !cli.local && (cli.gpu.is_some() || cli.leaderboard.is_some() || cli.mode.is_some()) { @@ -329,12 +359,21 @@ pub async fn execute(cli: Cli) -> Result<()> { // Handle the case where only a filepath is provided (for backward compatibility) if let Some(top_level_filepath) = cli.filepath { + let mode = if cli.profile || cli.profile_brev { + Some("profile".to_string()) + } else { + cli.mode + }; + cli.profile_options.validate(mode.as_deref())?; + if cli.local && is_profile_mode(mode.as_deref()) { + return Err(anyhow!("Profiling uses the hosted service; omit --local")); + } if cli.local { submit::run_submit_local( Some(top_level_filepath), cli.gpu, cli.leaderboard, - cli.mode, + mode, cli.output, ) .await @@ -350,14 +389,19 @@ pub async fn execute(cli: Cli) -> Result<()> { ) })?; - if cli.profile_brev { + if cli.profile_brev || is_profile_mode(mode.as_deref()) { submit::run_submit_plain( Some(top_level_filepath), - Some("B200_Brev".to_string()), + if cli.profile_brev { + Some("B200_Brev".to_string()) + } else { + cli.gpu + }, cli.leaderboard, Some("profile".to_string()), cli_id, - cli.benchmark_index, + cli.profile_options, + cli.profile_brev, cli.output, ) .await @@ -382,3 +426,64 @@ pub async fn execute(cli: Cli) -> Result<()> { } } } + +fn is_profile_mode(mode: Option<&str>) -> bool { + mode.is_some_and(|mode| mode.eq_ignore_ascii_case("profile")) +} + +#[cfg(test)] +mod profile_tests { + use super::*; + + #[tokio::test] + async fn local_rejects_profile_mode_inherited_from_top_level() { + let cli = Cli::try_parse_from([ + "popcorn", + "--mode", + "profile", + "submit", + "submission.py", + "--local", + ]) + .unwrap(); + let error = execute(cli).await.unwrap_err().to_string(); + assert!(error.contains("omit --local")); + } + + #[test] + fn profile_flags_accept_filters_at_both_entry_points() { + for prefix in [vec!["popcorn"], vec!["popcorn", "submit"]] { + for flag in ["--profile", "--profile-brev"] { + let mut args = prefix.clone(); + args.extend([ + "submission.py", + flag, + "--benchmark-index", + "3", + "--ncu-kernel-name", + "regex:custom", + "--ncu-launch-count", + "2", + ]); + assert!(Cli::try_parse_from(args).is_ok()); + } + } + assert!(Cli::try_parse_from([ + "popcorn", + "submit", + "submission.py", + "--profile", + "--profile-brev" + ]) + .is_err()); + assert!(Cli::try_parse_from([ + "popcorn", + "submit", + "submission.py", + "--profile", + "--ncu-kernel-name-base", + "invalid" + ]) + .is_err()); + } +} diff --git a/src/cmd/submit.rs b/src/cmd/submit.rs index ba5093e..bcc4582 100644 --- a/src/cmd/submit.rs +++ b/src/cmd/submit.rs @@ -681,13 +681,15 @@ pub async fn run_submit_tui( Ok(()) } +#[allow(clippy::too_many_arguments)] pub async fn run_submit_plain( filepath: Option, gpu: Option, leaderboard: Option, mode: Option, cli_id: String, - benchmark_index: Option, + profile_options: crate::profile::ProfileOptions, + profile_brev: bool, output: Option, ) -> Result<()> { let file_to_submit = match filepath { @@ -718,6 +720,11 @@ pub async fn run_submit_plain( None } }) + .or_else(|| { + mode.as_deref() + .is_some_and(|m| m.eq_ignore_ascii_case("profile")) + .then(|| "B200".to_string()) + }) .ok_or_else(|| anyhow!("GPU not specified. Use --gpu flag or add GPU directive to file"))?; let final_leaderboard = leaderboard @@ -749,20 +756,29 @@ pub async fn run_submit_plain( // Create client and submit let client = service::create_client(Some(cli_id))?; - let result = if final_mode.eq_ignore_ascii_case("profile") - && final_gpu.eq_ignore_ascii_case("B200_Brev") - { + let result = if final_mode.eq_ignore_ascii_case("profile") && profile_brev { service::profile_brev_solution( &client, &file_to_submit, &file_content, &final_leaderboard, - benchmark_index, + &profile_options, Some(Box::new(|msg| { eprintln!("{}", msg); })), ) .await? + } else if final_mode.eq_ignore_ascii_case("profile") { + service::profile_solution( + &client, + &file_to_submit, + &file_content, + &final_leaderboard, + &final_gpu, + &profile_options, + Some(Box::new(|msg| eprintln!("{}", msg))), + ) + .await? } else { service::submit_solution( &client, diff --git a/src/main.rs b/src/main.rs index 0b36a38..2f2d146 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,7 @@ mod cmd; mod local; mod models; +mod profile; mod service; mod utils; mod views; diff --git a/src/profile.rs b/src/profile.rs new file mode 100644 index 0000000..3616f3b --- /dev/null +++ b/src/profile.rs @@ -0,0 +1,240 @@ +use crate::service; +use anyhow::{anyhow, Context, Result}; +use base64::Engine; +use serde::Serialize; +use serde_json::Value; +use std::path::Path; + +#[derive(clap::Args, Debug, Default, Clone, Serialize)] +pub struct ProfileOptions { + /// Profile benchmarks[N]; omit to profile all benchmark shapes. + #[arg(long)] + pub benchmark_index: Option, + /// Capture only matching NCU kernel names (supports regex: expressions). + #[arg(long)] + pub ncu_kernel_name: Option, + /// How NCU interprets kernel names. + #[arg(long, value_parser = ["function", "demangled", "mangled"])] + pub ncu_kernel_name_base: Option, + /// Maximum matching kernel launches to capture per benchmark (default: 10). + #[arg(long)] + pub ncu_launch_count: Option, +} + +impl ProfileOptions { + pub fn merge(self, fallback: Self) -> Self { + Self { + benchmark_index: self.benchmark_index.or(fallback.benchmark_index), + ncu_kernel_name: self.ncu_kernel_name.or(fallback.ncu_kernel_name), + ncu_kernel_name_base: self.ncu_kernel_name_base.or(fallback.ncu_kernel_name_base), + ncu_launch_count: self.ncu_launch_count.or(fallback.ncu_launch_count), + } + } + + pub fn validate(&self, mode: Option<&str>) -> Result<()> { + if self.ncu_launch_count == Some(0) { + return Err(anyhow!("--ncu-launch-count must be greater than zero")); + } + if (self.benchmark_index.is_some() + || self.ncu_kernel_name.is_some() + || self.ncu_kernel_name_base.is_some() + || self.ncu_launch_count.is_some()) + && !mode.is_some_and(|mode| mode.eq_ignore_ascii_case("profile")) + { + return Err(anyhow!( + "Profiling options require --profile, --profile-brev, or --mode profile" + )); + } + Ok(()) + } +} + +fn run_failure(run: &Value) -> Option { + let compilation = run.get("compilation").filter(|value| !value.is_null()); + if compilation + .and_then(|value| value.get("success")) + .and_then(Value::as_bool) + == Some(false) + { + return Some(format!( + "Compilation failed:\n{}", + compilation + .and_then(|value| value.get("stderr")) + .and_then(Value::as_str) + .unwrap_or("No compiler error was reported") + )); + } + + let result = run.get("run")?; + if result.get("success").and_then(Value::as_bool) == Some(false) { + return Some(format!( + "Execution failed:\n{}", + result + .get("stderr") + .and_then(Value::as_str) + .unwrap_or("No execution error was reported") + )); + } + None +} + +fn save_profile_result(payload: &Value, output_dir: &Path) -> Result { + let result = &payload["result"]; + if result["success"].as_bool() != Some(true) { + return Err(anyhow!("Profiling failed: {}", result["error"])); + } + let runs = result["runs"] + .as_object() + .filter(|runs| !runs.is_empty()) + .ok_or_else(|| anyhow!("Server returned no profile runs"))?; + std::fs::create_dir_all(output_dir)?; + std::fs::write( + output_dir.join("manifest.json"), + serde_json::to_vec_pretty(&serde_json::json!({ + "leaderboard": payload["leaderboard"], + "gpu": payload["gpu"], + "system": result["system"], + "profile_metadata": result["profile_metadata"], + }))?, + )?; + let mut lines = vec![format!( + "Nsight Compute profile: {} on {}", + payload["leaderboard"].as_str().unwrap_or("unknown"), + payload["gpu"].as_str().unwrap_or("unknown") + )]; + let mut failures = Vec::new(); + for (index, (key, run)) in runs.iter().enumerate() { + if let Some(failure) = run_failure(run) { + failures.push(format!("{}: {}", key, failure)); + if let Some(stdout) = run.pointer("/run/stdout").and_then(Value::as_str) { + failures.push(stdout.to_string()); + } + } + let Some(trace) = run.pointer("/profile/trace").and_then(Value::as_str) else { + failures.push(format!("{}: No NCU report was produced", key)); + continue; + }; + let bytes = base64::engine::general_purpose::STANDARD + .decode(trace) + .context("Server returned an invalid profile archive")?; + // Use a locally generated filename, never a path from the remote response. + let zip_path = output_dir.join(format!("profile-{}.zip", index)); + std::fs::write(&zip_path, &bytes)?; + let extracted = service::extract_profile_artifacts(&zip_path, &bytes)?; + if extracted.reports.is_empty() { + failures.push(format!("{}: NCU archive contains no .ncu-rep report", key)); + } + lines.push(format!( + "{}: {}", + key, + run.pointer("/run/result/benchmark.0.spec") + .and_then(Value::as_str) + .unwrap_or("unknown benchmark") + )); + for path in extracted.details.iter().chain(extracted.reports.iter()) { + lines.push(format!(" {}", path.display())); + } + } + lines.push(format!( + "Manifest: {}", + output_dir.join("manifest.json").display() + )); + if !failures.is_empty() { + return Err(anyhow!("{}\n\n{}", lines.join("\n"), failures.join("\n"))); + } + Ok(lines.join("\n")) +} + +pub fn save_hosted_results(response: &Value, leaderboard: &str, gpu: &str) -> Result { + let results = response["results"] + .as_array() + .filter(|results| !results.is_empty()) + .ok_or_else(|| anyhow!("Server returned no profiling results"))?; + let output_dir = tempfile::Builder::new() + .prefix("popcorn-profile-") + .tempdir_in(std::env::current_dir()?)? + .into_path(); + let mut summaries = Vec::new(); + for (index, result) in results.iter().enumerate() { + summaries.push(save_profile_result( + &serde_json::json!({ + "leaderboard": leaderboard, "gpu": gpu, "result": result, + }), + &output_dir.join(format!("result-{}", index)), + )?); + } + Ok(summaries.join("\n\n")) +} + +#[cfg(test)] +mod tests { + use super::*; + fn profile_payload(trace: &str) -> Value { + serde_json::json!({ + "leaderboard": "qr_v2", "gpu": "B200", "result": { + "success": true, "runs": {"profile.3": { + "run": {"success": true, "result": {"benchmark.0.spec": "n: 512"}}, + "profile": {"trace": trace} + }} + } + }) + } + + #[test] + fn extracts_profile_artifacts_and_records_provenance() { + use std::io::{Cursor, Write}; + let mut zip = zip::ZipWriter::new(Cursor::new(Vec::new())); + for name in ["profile.ncu-rep", "ncu-details.txt", "ncu-details.csv"] { + zip.start_file( + format!("profile_data/{}", name), + zip::write::SimpleFileOptions::default(), + ) + .unwrap(); + zip.write_all(b"captured data").unwrap(); + } + let trace = + base64::engine::general_purpose::STANDARD.encode(zip.finish().unwrap().into_inner()); + let mut payload = profile_payload(&trace); + payload["result"]["profile_metadata"] = serde_json::json!({"config_sha256": "verified-sha", "capture_options": {"benchmark_index": 3}}); + payload["profile_options"] = serde_json::json!({"benchmark_index": 3}); + let directory = tempfile::tempdir().unwrap(); + let output = save_profile_result(&payload, directory.path()).unwrap(); + assert!(output.contains("profile.3: n: 512")); + assert_eq!( + std::fs::read(directory.path().join("profile-0/profile.ncu-rep")).unwrap(), + b"captured data" + ); + assert!(output.contains("ncu-details.csv")); + let manifest = std::fs::read_to_string(directory.path().join("manifest.json")).unwrap(); + assert!(manifest.contains("verified-sha")); + assert!(manifest.contains("benchmark_index")); + } + + #[test] + fn rejects_missing_reports_and_failed_runs() { + let directory = tempfile::tempdir().unwrap(); + let mut payload = profile_payload(""); + payload["result"]["runs"]["profile.3"]["profile"] = Value::Null; + payload["result"]["runs"]["profile.3"]["run"]["success"] = Value::Bool(false); + payload["result"]["runs"]["profile.3"]["run"]["stderr"] = serde_json::json!("NCU denied"); + let error = save_profile_result(&payload, directory.path()) + .unwrap_err() + .to_string(); + assert!(error.contains("NCU denied")); + assert!(error.contains("No NCU report")); + payload["result"]["runs"] = serde_json::json!({}); + assert!(save_profile_result(&payload, directory.path()).is_err()); + } + + #[test] + fn validates_profile_options_before_launch() { + let mut options = ProfileOptions { + benchmark_index: Some(0), + ..Default::default() + }; + assert!(options.validate(Some("benchmark")).is_err()); + assert!(options.validate(Some("profile")).is_ok()); + options.ncu_launch_count = Some(0); + assert!(options.validate(Some("profile")).is_err()); + } +} diff --git a/src/service/mod.rs b/src/service/mod.rs index d0c1ecc..96676bf 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -629,6 +629,7 @@ pub async fn submit_solution>( leaderboard, gpu, submission_mode, + None, on_log, ) .await; @@ -646,12 +647,35 @@ pub async fn submit_solution>( .await } +#[allow(clippy::too_many_arguments)] +pub async fn profile_solution>( + client: &Client, + filepath: P, + file_content: &[u8], + leaderboard: &str, + gpu: &str, + options: &crate::profile::ProfileOptions, + on_log: Option>, +) -> Result { + submit_solution_streaming( + client, + filepath, + file_content, + leaderboard, + gpu, + "profile", + Some(options), + on_log, + ) + .await +} + pub async fn profile_brev_solution>( client: &Client, filepath: P, file_content: &[u8], leaderboard: &str, - benchmark_index: Option, + options: &crate::profile::ProfileOptions, on_log: Option>, ) -> Result { let base_url = env::var("POPCORN_BREV_PROFILER_URL") @@ -663,6 +687,51 @@ pub async fn profile_brev_solution>( })?; let base_url = base_url.trim_end_matches('/'); + let requested_capture_options: Vec<&str> = [ + ("ncu_kernel_name", options.ncu_kernel_name.is_some()), + ( + "ncu_kernel_name_base", + options.ncu_kernel_name_base.is_some(), + ), + ("ncu_launch_count", options.ncu_launch_count.is_some()), + ] + .into_iter() + .filter_map(|(name, requested)| requested.then_some(name)) + .collect(); + if !requested_capture_options.is_empty() { + let health_resp = client + .get(format!("{}/health", base_url)) + .timeout(Duration::from_secs(30)) + .send() + .await?; + let health_status = health_resp.status(); + if !health_status.is_success() { + return Err(anyhow!( + "Profiler capability check returned status {}: {}", + health_status, + response_error_text(health_resp).await? + )); + } + let health: Value = health_resp.json().await?; + let supported = health + .pointer("/capabilities/request_capture_options") + .and_then(Value::as_array) + .ok_or_else(|| { + anyhow!( + "Profiler does not advertise request-scoped NCU capture; \ + refusing to silently run the default capture window" + ) + })?; + for name in &requested_capture_options { + if !supported.iter().any(|item| item.as_str() == Some(*name)) { + return Err(anyhow!( + "Profiler does not advertise request capture option {}", + name + )); + } + } + } + let filename = filepath .as_ref() .file_name() @@ -673,10 +742,20 @@ pub async fn profile_brev_solution>( let mut form = Form::new() .part("file", part) .text("leaderboard", leaderboard.to_string()); - if let Some(index) = benchmark_index { + if let Some(index) = options.benchmark_index { form = form.text("benchmark_index", index.to_string()); } + if let Some(value) = &options.ncu_kernel_name { + form = form.text("ncu_kernel_name", value.clone()); + } + if let Some(value) = &options.ncu_kernel_name_base { + form = form.text("ncu_kernel_name_base", value.clone()); + } + if let Some(value) = options.ncu_launch_count { + form = form.text("ncu_launch_count", value.to_string()); + } + let resp = client .post(format!("{}/profile", base_url)) .multipart(form) @@ -887,12 +966,15 @@ async fn download_profile_artifacts( } #[derive(Debug)] -struct ExtractedProfileArtifacts { - details: Vec, - reports: Vec, +pub(crate) struct ExtractedProfileArtifacts { + pub details: Vec, + pub reports: Vec, } -fn extract_profile_artifacts(zip_path: &Path, bytes: &[u8]) -> Result { +pub(crate) fn extract_profile_artifacts( + zip_path: &Path, + bytes: &[u8], +) -> Result { let mut archive = ZipArchive::new(Cursor::new(bytes)).map_err(|e| { anyhow!( "Failed to read profile artifact {}: {}", @@ -1553,6 +1635,7 @@ fn format_submission_details(details: &SubmissionDetails) -> Result { } } +#[allow(clippy::too_many_arguments)] async fn submit_solution_streaming>( client: &Client, filepath: P, @@ -1560,6 +1643,7 @@ async fn submit_solution_streaming>( leaderboard: &str, gpu: &str, submission_mode: &str, + profile_options: Option<&crate::profile::ProfileOptions>, on_log: Option>, ) -> Result { let base_url = @@ -1573,15 +1657,38 @@ async fn submit_solution_streaming>( let part = Part::bytes(file_content.to_vec()).file_name(filename.to_string()); - let form = Form::new().part("file", part); + let mut form = Form::new().part("file", part); + if let Some(options) = profile_options { + options.validate(Some("profile"))?; + for (key, value) in serde_json::to_value(options)?.as_object().unwrap() { + if !value.is_null() { + form = form.text( + key.clone(), + value + .as_str() + .map(str::to_owned) + .unwrap_or_else(|| value.to_string()), + ); + } + } + } - let url = format!( - "{}/{}/{}/{}", - base_url, - leaderboard.to_lowercase(), - gpu, - submission_mode.to_lowercase() - ); + let url = if profile_options.is_some() { + format!( + "{}/profile/{}/{}", + base_url, + leaderboard.to_lowercase(), + gpu + ) + } else { + format!( + "{}/{}/{}/{}", + base_url, + leaderboard.to_lowercase(), + gpu, + submission_mode.to_lowercase() + ) + }; let resp = client .post(&url) @@ -1648,6 +1755,13 @@ async fn submit_solution_streaming>( } "result" => { let result_val: Value = serde_json::from_str(data)?; + if profile_options.is_some() { + return crate::profile::save_hosted_results( + &result_val, + leaderboard, + gpu, + ); + } if let Some(ref cb) = on_log { // Handle "results" array @@ -1758,6 +1872,9 @@ async fn submit_solution_streaming>( )) } else { let result: Value = resp.json().await?; + if profile_options.is_some() { + return crate::profile::save_hosted_results(&result, leaderboard, gpu); + } let pretty_result = match result.get("results") { Some(result_obj) => serde_json::to_string_pretty(result_obj)?, None => return Err(anyhow!("Invalid non-streaming response structure")), @@ -2039,6 +2156,56 @@ mod tests { String::from_utf8(request).unwrap() } + #[tokio::test] + async fn hosted_profile_uses_popcorn_auth_and_request_options() { + let _env_guard = ENV_LOCK.lock().await; + let original = std::env::var("POPCORN_API_URL").ok(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + std::env::set_var( + "POPCORN_API_URL", + format!("http://{}", listener.local_addr().unwrap()), + ); + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + let request = read_http_request(&mut stream).await; + assert!(request.starts_with("POST /profile/qr_v2/B200 HTTP/1.1")); + assert!(request + .to_lowercase() + .contains("x-popcorn-cli-id: test-user")); + for field in ["benchmark_index", "ncu_kernel_name", "ncu_launch_count"] { + assert!(request.contains(&format!("name=\"{}\"", field))); + } + assert!(request.contains("regex:solver")); + let body = "event: error\ndata: {\"detail\":\"deliberate capture failure\"}\n\n"; + stream.write_all(format!("HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", body.len(), body).as_bytes()).await.unwrap(); + }); + let client = create_client(Some("test-user".to_string())).unwrap(); + let result = profile_solution( + &client, + "submission.py", + b"pass", + "qr_v2", + "B200", + &crate::profile::ProfileOptions { + benchmark_index: Some(1), + ncu_kernel_name: Some("regex:solver".to_string()), + ncu_launch_count: Some(2), + ..Default::default() + }, + None, + ) + .await; + server.await.unwrap(); + match original { + Some(value) => std::env::set_var("POPCORN_API_URL", value), + None => std::env::remove_var("POPCORN_API_URL"), + } + assert!(result + .unwrap_err() + .to_string() + .contains("deliberate capture failure")); + } + #[tokio::test] async fn test_standard_submission_modes_use_background_results_except_profile() { let _env_guard = ENV_LOCK.lock().await;