diff --git a/docs/hosted-ncu-profiling.md b/docs/hosted-ncu-profiling.md new file mode 100644 index 00000000..e11d1285 --- /dev/null +++ b/docs/hosted-ncu-profiling.md @@ -0,0 +1,79 @@ +# Hosted Nsight Compute profiling + +`popcorn submit submission.py --leaderboard qr_v2 --profile --benchmark-index 0` +uses the normal Popcorn identity and API. Users do not install Modal, create a +provider account, or supply provider credentials. KernelBot dispatches the job +to its GPU runner and streams back reports. `--profile-brev` remains an explicit, +separate CLI choice; failures never switch providers. + +## API and capture contract + +`POST /profile/{leaderboard_name}/{gpu_type}` accepts the usual +`X-Popcorn-Cli-Id` header and a multipart `file`, plus optional fields: + +| Field | Meaning | +| --- | --- | +| `benchmark_index` | Zero-based index in the task's `benchmarks`; omit for all entries. | +| `ncu_kernel_name` | NCU kernel filter, including `regex:` expressions. | +| `ncu_kernel_name_base` | `function`, `demangled`, or `mangled`. | +| `ncu_launch_count` | Positive capture limit per benchmark; default 10. | + +The route retains normal submission permissions, limits, and leaderboard GPU +validation. It supports the configured single-GPU NVIDIA Modal runners. Invalid +options and unsupported GPUs are rejected; unsupported evaluators fail with NCU +diagnostics. The dedicated route prevents older API versions from silently +ignoring capture options. + +The runner sets `POPCORN_NCU=1`, follows child processes, and captures the NVTX +push/pop range `custom_kernel/`. It collects the full NCU section set with kernel +replay and leaves GPU clocks unchanged. Each benchmark returns a zip containing +`profile.ncu-rep`, `ncu-details.txt`, and `ncu-details.csv`. A missing capture is a +failure, even when the profiler process exits successfully. + +`FullResult.profile_metadata` records selected benchmark specifications, capture +options, and a SHA-256 fingerprint of source/configuration fields. This is not a +git commit. Record the synced reference-kernels revision in operator validation +records; the client cannot update the hosted task through a local checkout. + +## Rollout + +Deploy the updated GPU runners and API before releasing the companion +[popcorn-cli change](https://github.com/gpu-mode/popcorn-cli/pull/80). The runner +image pins NCU 2025.2.1 and checks `ncu --version` during its build. NCU 2026.2 +from CUDA 13.3 produced many NaN counters on the tested Modal B200; repeating +the same capture with 2025.2.1 restored hardware counters. Use the existing Modal deployment +workflow and API deployment process; no additional container privileges or +user-facing provider setup is introduced. + +Sync an NCU-compatible task through the existing problem-update workflow, then +validate one benchmark through the public API using an ordinary Popcorn user. +Inspect the returned report and counter exports. The +[reference-kernels guide](https://github.com/gpu-mode/reference-kernels/pull/171) +describes the evaluator contract for new problems. Merely accepting `profile` +and running `torch.profiler` does not satisfy that contract. + +## Validation + +CPU tests cover API authentication and options, benchmark selection, capture +commands, exports, and empty captures. The GPU fixture is +`problems/linalg/qr_v2`, benchmark 0 (`batch=20, n=32, cond=1, seed=43214`), at +reference-kernels `51e22db671d36c1c76091c43c36a44546ba324a1`. +Only this problem/shape is used for the GPU integration check; other tasks and +GPU types require their own validation. + +The integration check used the real FastAPI route, submission preparation, +KernelBackend, and GPU `run_config`, with a fake database and an ephemeral +launcher instead of production infrastructure. The CLI used normal Popcorn +header authentication with an empty `PATH` and no `MODAL_*` environment values. +It successfully saved and extracted a 7,593,257-byte report and both detail +exports. NCU reopened the report to produce those exports. + +NCU 2025.2.1 collected 39 passes on NVIDIA B200 with PyTorch 2.12.0+cu130, +CUDA 13.3, and `regex:geqr2` / demangled / launch count 1. Observed counters: +322.78 us duration, 12.51% achieved occupancy, 0.42% SM throughput, 70.09% L2 +hit rate, and 1,565,428 executed instructions. Six `ctc__*` metrics remained +unavailable. This validates the isolated implementation; the production API +has not been deployed as part of this change. + +[Ephemeral validation run](https://modal.com/apps/coreauto/main/ap-p8eodYfHkZAMp0ZT5btR1y) +(access requires the operator's workspace permissions). diff --git a/src/kernelbot/api/main.py b/src/kernelbot/api/main.py index 7a7d4e6e..52c2b5f9 100644 --- a/src/kernelbot/api/main.py +++ b/src/kernelbot/api/main.py @@ -8,17 +8,18 @@ from dataclasses import asdict from typing import Annotated, Any, Optional -from fastapi import Depends, FastAPI, Header, HTTPException, Query, Request, UploadFile +from fastapi import Depends, FastAPI, Form, Header, HTTPException, Query, Request, UploadFile from fastapi.responses import JSONResponse, StreamingResponse from kernelbot.env import env from libkernelbot.application_validation import ApplicationValidationService from libkernelbot.backend import KernelBackend from libkernelbot.background_submission_manager import BackgroundSubmissionManager -from libkernelbot.consts import SubmissionMode +from libkernelbot.consts import SubmissionMode, get_gpu_by_name from libkernelbot.db_types import IdentityType from libkernelbot.leaderboard_db import LeaderboardDB, LeaderboardRankedEntry from libkernelbot.problem_sync import sync_problems +from libkernelbot.profiling import ProfileOptions from libkernelbot.submission import ( ProcessedSubmissionRequest, SubmissionRequest, @@ -522,6 +523,43 @@ async def admin_run_application_validation( } +@app.post("/profile/{leaderboard_name}/{gpu_type}") +async def profile_submission( + leaderboard_name: str, + gpu_type: str, + file: UploadFile, + user_info: Annotated[dict, Depends(validate_cli_header)], + benchmark_index: Annotated[int | None, Form(ge=0)] = None, + ncu_kernel_name: Annotated[str | None, Form(min_length=1, max_length=1024)] = None, + ncu_kernel_name_base: Annotated[str | None, Form()] = None, + ncu_launch_count: Annotated[int | None, Form(ge=1)] = None, + db_context=Depends(get_db), +) -> StreamingResponse: + """Profile through the normal authenticated submission pipeline. + + A dedicated route prevents older API deployments from silently ignoring + capture options sent by a newer CLI. Compute-provider credentials stay here. + """ + await simple_rate_limit() + try: + gpu = get_gpu_by_name(gpu_type) + if gpu is None or gpu.runner != "Modal" or gpu.name == "L4x4": + raise ValueError("Hosted NCU profiling requires a supported single NVIDIA GPU") + options = ProfileOptions( + benchmark_index, ncu_kernel_name, ncu_kernel_name_base, ncu_launch_count + ) + except (ValueError, KeyError) as error: + raise HTTPException(status_code=400, detail=str(error)) from error + request, mode = await to_submit_info( + user_info, "profile", file, leaderboard_name, gpu_type, db_context + ) + request.profile_options = options.to_dict() + return StreamingResponse( + _stream_submission_response(request, mode, backend_instance), + media_type="text/event-stream", + ) + + @app.post("/{leaderboard_name}/{gpu_type}/{submission_mode}") async def run_submission( # noqa: C901 leaderboard_name: str, diff --git a/src/libkernelbot/backend.py b/src/libkernelbot/backend.py index 50a3d262..e47986c9 100644 --- a/src/libkernelbot/backend.py +++ b/src/libkernelbot/backend.py @@ -136,6 +136,10 @@ async def submit_full( req.task, mode, None, + **( + {"profile_options": req.profile_options} + if req.profile_options is not None else {} + ), ) for gpu in selected_gpus ] @@ -174,6 +178,7 @@ async def submit_leaderboard( # noqa: C901 task: LeaderboardTask, mode: SubmissionMode, seed: Optional[int], + profile_options: dict | None = None, ) -> Optional[FullResult]: """ Function invoked by `leaderboard_cog` to handle a leaderboard run. @@ -193,6 +198,7 @@ async def submit_leaderboard( # noqa: C901 task=task, mode=mode, submission_id=submission_id, + **({"profile_options": profile_options} if profile_options is not None else {}), ) if result.success: @@ -232,6 +238,7 @@ async def handle_submission( task: Optional[LeaderboardTask], mode: SubmissionMode, submission_id: int = -1, + profile_options: dict | None = None, ) -> Optional[FullResult]: """ Generic function to handle code submissions. @@ -250,6 +257,9 @@ async def handle_submission( task=task, submission_content=code, arch=self._get_arch(gpu_type), mode=mode ) + if profile_options is not None: + config["profile_options"] = profile_options + logger.info("submitting task to runner %s", launcher.name) result = await launcher.run_submission(config, gpu_type, reporter) diff --git a/src/libkernelbot/profiling.py b/src/libkernelbot/profiling.py new file mode 100644 index 00000000..5fb2f628 --- /dev/null +++ b/src/libkernelbot/profiling.py @@ -0,0 +1,38 @@ +"""Validated, request-scoped profiling options shared by API and GPU runners.""" + +from dataclasses import asdict, dataclass +from typing import Literal + + +@dataclass(frozen=True) +class ProfileOptions: + benchmark_index: int | None = None + ncu_kernel_name: str | None = None + ncu_kernel_name_base: Literal["function", "demangled", "mangled"] | None = None + ncu_launch_count: int | None = None + + def __post_init__(self): + for name, minimum in [("benchmark_index", 0), ("ncu_launch_count", 1)]: + value = getattr(self, name) + if value is not None and (type(value) is not int or value < minimum): + raise ValueError(f"{name} must be an integer >= {minimum}") + if self.ncu_kernel_name_base not in (None, "function", "demangled", "mangled"): + raise ValueError("ncu_kernel_name_base must be function, demangled, or mangled") + if self.ncu_kernel_name is not None and ( + not isinstance(self.ncu_kernel_name, str) + or not self.ncu_kernel_name + or len(self.ncu_kernel_name) > 1024 + or "\x00" in self.ncu_kernel_name + ): + raise ValueError("ncu_kernel_name must contain 1..1024 characters without NUL") + + def validate_task(self, benchmarks: list, multi_gpu: bool): + if multi_gpu: + raise ValueError("NCU profiling requires a single GPU") + if not benchmarks: + raise ValueError("This task has no benchmarks to profile") + if self.benchmark_index is not None and self.benchmark_index >= len(benchmarks): + raise ValueError(f"benchmark_index must be between 0 and {len(benchmarks) - 1}") + + def to_dict(self) -> dict: + return {name: value for name, value in asdict(self).items() if value is not None} diff --git a/src/libkernelbot/run_eval.py b/src/libkernelbot/run_eval.py index dfabd24e..3fc7df48 100644 --- a/src/libkernelbot/run_eval.py +++ b/src/libkernelbot/run_eval.py @@ -3,6 +3,7 @@ import dataclasses import datetime import functools +import hashlib import json import os import shlex @@ -16,6 +17,7 @@ from typing import Optional, Protocol, Union from libkernelbot.consts import CUDA_FLAGS, ExitCode, Timeout +from libkernelbot.profiling import ProfileOptions @dataclasses.dataclass @@ -102,6 +104,7 @@ class FullResult: # 'test' and 'benchmark' keys, for example runs: dict[str, EvalResult] = dataclasses.field(default_factory=dict) cpu_compile: CPUCompileInfo | None = None + profile_metadata: dict | None = None # fmt: on @@ -465,60 +468,55 @@ def profile_program_ncu( timeout: int, multi_gpu: bool, output_dir: Path, + options: dict | None = None, ) -> tuple[RunResult, Optional[ProfileResult]]: - assert not multi_gpu, "Multi-GPU profiling not supported for ncu." - - # Wrap program in ncu - call = [ - "ncu", - "--set", - "full", - "--nvtx", - "--nvtx-include", - "custom_kernel/", - "--import-source", - "1", - "-c", - "10", - "-o", - f"{str(output_dir / 'profile.ncu-rep')}", - "--", - ] + call - + if multi_gpu: + raise ValueError("Multi-GPU profiling not supported for ncu") + capture = ProfileOptions(**(options or {})) + command = [ + "ncu", "--set", "full", "--target-processes", "all", + "--nvtx", "--nvtx-include", "custom_kernel/", "--import-source", "1", + "--launch-count", str(capture.ncu_launch_count or 10), + "--cache-control", "all", "--clock-control", "none", + "--replay-mode", "kernel", "--force-overwrite", + "--export", str(output_dir / "profile.ncu-rep"), + ] + if capture.ncu_kernel_name is not None: + command += ["--kernel-name", capture.ncu_kernel_name] + if capture.ncu_kernel_name_base is not None: + command += ["--kernel-name-base", capture.ncu_kernel_name_base] run_result = run_program( - call, seed=seed, timeout=timeout, multi_gpu=multi_gpu, extra_env={"POPCORN_NCU": "1"} + command + ["--", *call], seed=seed, timeout=timeout, multi_gpu=False, + extra_env={"POPCORN_NCU": "1"}, ) - profile_result = None - - try: - get_tables = [ - "GPU Throughput", - "Pipe Utilization (% of active cycles)", - "Warp State (All Cycles)", - ] - ncu_cmd = [ - "ncu", - "--import", - f"{str(output_dir / 'profile.ncu-rep')}", - "--print-details", - "body", - ] - report = subprocess.check_output(ncu_cmd, text=True) - report = _filter_ncu_report(report, get_tables) - run_result.result["benchmark.0.report"] = base64.b64encode(report.encode("utf-8")).decode( - "utf-8" + report_path = output_dir / "profile.ncu-rep" + if not run_result.success or not report_path.is_file(): + run_result.success = False + run_result.stderr += ( + "\nNCU did not produce a report. Check stdout, the kernel filter, " + "and profile-mode NVTX support." ) - except subprocess.CalledProcessError: - pass + return run_result, None - if run_result.success: - profile_result = ProfileResult( - profiler="Nsight-Compute", - trace=_directory_to_zip_bytes(output_dir), - download_url=None, + for name, extra in [("ncu-details.txt", []), ("ncu-details.csv", ["--csv"])]: + details = subprocess.run( + ["ncu", "--import", str(report_path), "--page", "details", *extra], + capture_output=True, text=True, timeout=120, ) - - return run_result, profile_result + if details.returncode: + run_result.success = False + run_result.stderr += f"\nFailed to export {name}: {details.stderr}" + else: + (output_dir / name).write_text(details.stdout) + text_path = output_dir / "ncu-details.txt" + if text_path.exists(): + summary = _filter_ncu_report(text_path.read_text(), [ + "GPU Throughput", "Pipe Utilization (% of active cycles)", "Warp State (All Cycles)", + ]) + run_result.result["benchmark.0.report"] = base64.b64encode(summary.encode()).decode() + return run_result, ProfileResult( + profiler="Nsight-Compute", trace=_directory_to_zip_bytes(output_dir), download_url=None, + ) def profile_program( @@ -527,6 +525,7 @@ def profile_program( seed: Optional[int], timeout: int, multi_gpu: bool, + options: dict | None = None, ) -> tuple[RunResult, Optional[ProfileResult]]: # The runner-specific configuration should implement logic # to fetch the data in this directory and return it as @@ -540,7 +539,7 @@ def profile_program( if system.runtime == "ROCm": return profile_program_roc(call, seed, timeout, multi_gpu, output_dir) elif system.runtime == "CUDA": - return profile_program_ncu(call, seed, timeout, multi_gpu, output_dir) + return profile_program_ncu(call, seed, timeout, multi_gpu, output_dir, options) else: raise ValueError(f"Unknown runtime {system.runtime}") @@ -558,6 +557,7 @@ def run_single_evaluation( ranked_timeout: int = Timeout.RANKED, ranking_by: str = "last", seed: Optional[int] = None, + profile_options: dict | None = None, ) -> tuple[RunResult, Optional[ProfileResult]]: """ A single runner run, either in the context of test files, or in the @@ -581,7 +581,10 @@ def run_single_evaluation( call = call + [mode, cases.name] if mode == "profile": - return profile_program(system, call, seed=seed, timeout=timeout, multi_gpu=multi_gpu) + return profile_program( + system, call, seed=seed, timeout=timeout, + multi_gpu=multi_gpu, options=profile_options, + ) return run_program(call, seed=seed, timeout=timeout, multi_gpu=multi_gpu), None @@ -797,6 +800,7 @@ def run_evaluation( call: _EvalRunner, mode: str, common_args: dict, + benchmark_index: int | None = None, ) -> dict[str, EvalResult]: """ Given a "runner" function `call`, interprets the mode @@ -809,6 +813,8 @@ def run_evaluation( if mode == "profile": benchmarks = copy.deepcopy(common_args["benchmarks"]) for i, benchmark in enumerate(benchmarks.splitlines()): + if benchmark_index is not None and i != benchmark_index: + continue common_args["benchmarks"] = benchmark results[f"{mode}.{i}"] = call(mode=mode, **common_args) @@ -846,6 +852,9 @@ def build_test_string(tests: list[dict]): def run_config(config: dict): system = make_system_info() + profile_options = ProfileOptions(**config.get("profile_options", {})) + if config["mode"] == "profile" and system.runtime == "CUDA": + profile_options.validate_task(config.get("benchmarks", []), config.get("multi_gpu", False)) common_args = { "system": system, "tests": build_test_string(config.get("tests", [])), @@ -856,6 +865,7 @@ def run_config(config: dict): "benchmark_timeout": config.get("benchmark_timeout", Timeout.BENCHMARK), "test_timeout": config.get("test_timeout", Timeout.TEST), "multi_gpu": config.get("multi_gpu", False), + "profile_options": profile_options.to_dict(), } if config["lang"] == "py": runner = functools.partial( @@ -875,5 +885,22 @@ def run_config(config: dict): else: raise ValueError(f"Invalid language {config['lang']}") - results = run_evaluation(runner, config["mode"], common_args) - return FullResult(success=True, error="", runs=results, system=system) + results = run_evaluation(runner, config["mode"], common_args, profile_options.benchmark_index) + metadata = None + if config["mode"] == "profile": + metadata = { + "capture_options": profile_options.to_dict(), + "benchmark_specs": { + str(i): shape for i, shape in enumerate(config.get("benchmarks", [])) + if profile_options.benchmark_index is None or i == profile_options.benchmark_index + }, + "config_sha256": hashlib.sha256(json.dumps({ + key: config.get(key) for key in ( + "lang", "main", "sources", "headers", "arch", + "tests", "benchmarks", "profile_options" + ) + }, sort_keys=True).encode()).hexdigest(), + } + return FullResult( + success=True, error="", runs=results, system=system, profile_metadata=metadata, + ) diff --git a/src/libkernelbot/submission.py b/src/libkernelbot/submission.py index a1c8df64..7b39a946 100644 --- a/src/libkernelbot/submission.py +++ b/src/libkernelbot/submission.py @@ -10,6 +10,7 @@ from libkernelbot.consts import RankCriterion, SubmissionMode, get_mode_category from libkernelbot.db_types import RunItem, SubmissionItem from libkernelbot.leaderboard_db import LeaderboardDB, LeaderboardItem +from libkernelbot.profiling import ProfileOptions from libkernelbot.run_eval import FullResult from libkernelbot.task import LeaderboardTask from libkernelbot.utils import KernelBotError, format_time, setup_logging @@ -31,6 +32,7 @@ class SubmissionRequest: gpus: Union[None, str, list] leaderboard: Optional[str] identity_type: Optional[str] = None + profile_options: dict | None = None @dataclasses.dataclass @@ -109,6 +111,16 @@ def prepare_submission( # noqa: C901 elif len(task_gpus) == 1: req.gpus = task_gpus + if req.profile_options is not None: + if mode != SubmissionMode.PROFILE: + raise KernelBotError("Profile options require profile mode") + try: + options = ProfileOptions(**req.profile_options) + options.validate_task(leaderboard["task"].benchmarks, leaderboard["task"].multi_gpu) + req.profile_options = options.to_dict() + except (ValueError, TypeError) as error: + raise KernelBotError(str(error)) from error + return ProcessedSubmissionRequest( **dataclasses.asdict(req), task=leaderboard["task"], diff --git a/src/runners/modal_runner.py b/src/runners/modal_runner.py index 4a4c01b1..a457a44c 100644 --- a/src/runners/modal_runner.py +++ b/src/runners/modal_runner.py @@ -136,6 +136,13 @@ .env({"CXX": "/opt/kernelbot-pch/compiler.py", "KERNELBOT_PCH_VOLUME": PCH_VOLUME_NAME}) ) +# NCU 2026.2 in CUDA 13.3 returned NaN hardware counters on Modal B200. +# Keep the tested profiler until a newer version passes the same capture check. +cuda_image = cuda_image.apt_install("nsight-compute-2025.2.1").run_commands( + "ln -sf /opt/nvidia/nsight-compute/2025.2.1/ncu $(command -v ncu)", + "ncu --version", +) + cuda_image = cuda_image.add_local_python_source( "libkernelbot", "modal_runner", diff --git a/tests/test_profiling.py b/tests/test_profiling.py new file mode 100644 index 00000000..f6bfe89e --- /dev/null +++ b/tests/test_profiling.py @@ -0,0 +1,174 @@ +"""Hosted profiling contract tests; no cloud account or GPU is required.""" + +import base64 +import io +import zipfile +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from fastapi.testclient import TestClient + +from libkernelbot import run_eval +from libkernelbot.consts import SubmissionMode +from libkernelbot.profiling import ProfileOptions +from libkernelbot.submission import SubmissionRequest + + +@pytest.mark.parametrize( + "values", + [ + {"benchmark_index": -1}, + {"benchmark_index": True}, + {"ncu_launch_count": 0}, + {"ncu_kernel_name_base": "invalid"}, + {"ncu_kernel_name": ""}, + ], +) +def test_invalid_capture_options(values): + with pytest.raises(ValueError): + ProfileOptions(**values) + + +def test_reject_out_of_range_and_multi_gpu(): + with pytest.raises(ValueError): + ProfileOptions(benchmark_index=2).validate_task([{"n": 32}], False) + with pytest.raises(ValueError): + ProfileOptions().validate_task([{"n": 32}], True) + + +def test_runner_selects_original_benchmark_index_without_mutating_config(): + config = { + "lang": "py", + "sources": {"eval.py": "pass"}, + "main": "eval.py", + "mode": "profile", + "benchmarks": [{"n": 32}, {"n": 512}], + "profile_options": {"benchmark_index": 1, "ncu_kernel_name": "regex:solver"}, + "cpu_compile": {"artifacts": b"not-json"}, + } + with ( + patch.object( + run_eval, "make_system_info", return_value=run_eval.SystemInfo(runtime="CUDA") + ), + patch.object(run_eval, "run_pytorch_script", return_value="captured") as runner, + ): + result = run_eval.run_config(config) + assert result.runs == {"profile.1": "captured"} + assert runner.call_args.kwargs["benchmarks"] == "n: 512" + assert runner.call_args.kwargs["profile_options"]["ncu_kernel_name"] == "regex:solver" + assert len(config["benchmarks"]) == 2 + assert result.profile_metadata["benchmark_specs"] == {"1": {"n": 512}} + assert len(result.profile_metadata["config_sha256"]) == 64 + + +def test_ncu_captures_children_and_exports_reports(tmp_path): + result = SimpleNamespace(success=True, stderr="", result={}) + + def capture(command, **kwargs): + assert command[command.index("--clock-control") + 1] == "none" + assert command[command.index("--target-processes") + 1] == "all" + assert command[command.index("--kernel-name") + 1] == "regex:solver|a b" + assert command[command.index("--launch-count") + 1] == "2" + assert kwargs["extra_env"] == {"POPCORN_NCU": "1"} + (tmp_path / "profile.ncu-rep").write_bytes(b"report") + return result + + with ( + patch.object(run_eval, "run_program", side_effect=capture), + patch.object( + run_eval.subprocess, + "run", + return_value=SimpleNamespace(returncode=0, stdout="counter data", stderr=""), + ), + ): + run, profile = run_eval.profile_program_ncu( + ["python3", "eval.py"], + None, + 30, + False, + tmp_path, + {"ncu_kernel_name": "regex:solver|a b", "ncu_launch_count": 2}, + ) + assert run.success + with zipfile.ZipFile(io.BytesIO(base64.b64decode(profile.trace))) as archive: + names = {name.split("/")[-1] for name in archive.namelist()} + assert {"profile.ncu-rep", "ncu-details.txt", "ncu-details.csv"} <= names + + +def test_empty_ncu_capture_is_a_failure(tmp_path): + result = SimpleNamespace(success=True, stderr="", stdout="No kernels profiled", result={}) + with patch.object(run_eval, "run_program", return_value=result): + run, profile = run_eval.profile_program_ncu(["eval"], None, 30, False, tmp_path) + assert not run.success + assert profile is None + assert "did not produce a report" in run.stderr + assert run.stdout == "No kernels profiled" + + +def test_profile_endpoint_requires_normal_cli_auth(): + from kernelbot.api.main import app, get_db + + app.dependency_overrides[get_db] = lambda: MagicMock() + try: + response = TestClient(app).post( + "/profile/qr_v2/B200", files={"file": ("submission.py", b"pass")} + ) + assert response.status_code == 400 + assert "Missing X-Popcorn-Cli-Id" in response.json()["detail"] + finally: + app.dependency_overrides.clear() + + +def test_profile_endpoint_preserves_options_and_validates_input(): + from kernelbot.api import main + + captured = [] + + async def stream(request, mode, backend): + captured.append((request, mode)) + yield 'event: result\ndata: {"results": []}\n\n' + + request = SubmissionRequest("pass", "submission.py", 1, "test", ["B200"], "qr_v2") + main.app.dependency_overrides[main.validate_cli_header] = lambda: { + "user_id": 1, + "user_name": "test", + } + main.app.dependency_overrides[main.get_db] = lambda: MagicMock() + try: + with ( + patch.object( + main, "to_submit_info", AsyncMock(return_value=(request, SubmissionMode.PROFILE)) + ), + patch.object(main, "_stream_submission_response", stream), + patch.object(main, "simple_rate_limit", AsyncMock()), + ): + client = TestClient(main.app) + response = client.post( + "/profile/qr_v2/B200", + files={"file": ("submission.py", b"pass")}, + data={ + "benchmark_index": "1", + "ncu_kernel_name": "regex:solver", + "ncu_launch_count": "2", + }, + ) + assert response.status_code == 200 + assert captured[0][0].profile_options == { + "benchmark_index": 1, + "ncu_kernel_name": "regex:solver", + "ncu_launch_count": 2, + } + assert captured[0][1] == SubmissionMode.PROFILE + for data, status in [ + ({"benchmark_index": "-1"}, 422), + ({"ncu_launch_count": "0"}, 422), + ({"ncu_kernel_name_base": "invalid"}, 400), + ]: + response = client.post( + "/profile/qr_v2/B200", files={"file": ("submission.py", b"pass")}, data=data + ) + assert response.status_code == status + assert len(captured) == 1 + finally: + main.app.dependency_overrides.clear()