feat(personaplex): qualify spoken QA with AIPerf

apps/aiperf_qual/config/suites/librispeech-test-clean-speech-20.yaml

@@ -3,14 +3,16 @@
 
 # LibriSpeech test-clean (AIPerf `librispeech`, pinned): 20 utterances in seeded order, one per speaker, as the user
 # turn of a speech-to-speech model; no gold: the spoken answer is compared with the native model's.
+# Version 2 resamples the 16 kHz utterances once to the model's 24 kHz input for both sides.
 suite: librispeech-test-clean-speech-20
-version: 1
+version: 2
 source:
   kind: aiperf_public
   dataset: librispeech
   label_column: text
   cluster_column: speaker_id
   one_per_cluster: true
+  audio_sample_rate: 24000
 selection: {method: seeded, count: 20}
 fields: {audio: audio_path}
 request:

apps/aiperf_qual/tests/test_output_postprocess.py

@@ -0,0 +1,37 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import json
+import sys
+
+import pytest
+
+from trtmc_aiperf_qual.absolute import postprocess
+from trtmc_aiperf_qual.config import Environment
+
+
+def test_postprocess_adds_text_without_changing_timing_or_original_artifacts(tmp_path):
+    script = tmp_path / "transcribe.py"
+    script.write_text("import json,sys\nfrom pathlib import Path\n"
+                      "d=json.loads(Path(sys.argv[2]).read_text())\n"
+                      "for row in d.values(): row['text']='Option C'\n"
+                      "Path(sys.argv[4]).write_text(json.dumps(d))\n")
+    observations = {0: {"artifact": "unchanged.npy", "sample_rate": 24000}}
+    item = {"postprocess": {"script": script.name}}
+    environment = Environment({"repo": str(tmp_path), "serve_python": "/unused-serving-interpreter"})
+    postprocess(environment, item, observations, tmp_path / "run", python=sys.executable)
+    assert observations == {0: {"artifact": "unchanged.npy", "sample_rate": 24000, "text": "Option C"}}
+    assert json.loads((tmp_path / "run.postprocess-input.json").read_text()) == {
+        "0": {"artifact": "unchanged.npy", "sample_rate": 24000}}
+
+
+@pytest.mark.parametrize("mutation", ["d.clear()", "d['0']['sample_rate']=16000"])
+def test_postprocess_rejects_lost_samples_or_modified_evidence(tmp_path, mutation):
+    script = tmp_path / "bad.py"
+    script.write_text("import json,sys\nfrom pathlib import Path\n"
+                      "d=json.loads(Path(sys.argv[2]).read_text())\n" + mutation + "\n"
+                      "Path(sys.argv[4]).write_text(json.dumps(d))\n")
+    environment = Environment({"repo": str(tmp_path), "serve_python": sys.executable})
+    with pytest.raises(ValueError, match="postprocessing changed"):
+        postprocess(environment, {"postprocess": {"script": script.name}},
+                    {0: {"sample_rate": 24000}}, tmp_path / "run")

apps/aiperf_qual/tests/test_spoken_choice.py

@@ -0,0 +1,46 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+"""Spoken answer scoring must not reward silence or failed extraction."""
+
+import pytest
+
+from trtmc_aiperf_qual.gold_metrics import spoken_choice
+
+GOLD = {"answer": "C", "prompt": "Predators eat\nA. lions\nB. humans\nC. bunnies\nD. grass\nSelect A, B, C, or D."}
+
+
+@pytest.mark.parametrize("text", ["Option C, bunnies.", "C.", "The answer is C.", "They eat bunnies."])
+def test_recognizes_the_correct_spoken_answer(text):
+    assert spoken_choice(GOLD, {"text": text}) == (True, "C")
+
+
+@pytest.mark.parametrize("text", ["", "Thank you for listening.", "A predator needs food.",
+                                  "Option A or option C.", "lions and bunnies"])
+def test_missing_or_ambiguous_answers_are_never_guessed(text):
+    assert spoken_choice(GOLD, {"text": text}) == (False, "")
+
+
+def test_wrong_answers_remain_wrong():
+    assert spoken_choice(GOLD, {"text": "Option D, grass."}) == (False, "D")
+
+
+def test_an_explicit_answer_overrides_a_word_in_the_explanation():
+    assert spoken_choice(GOLD, {"text": "Bunnies are animals. The answer is A. Okay."}) == (False, "A")
+
+
+@pytest.mark.parametrize("text", ["I would say C, definitely.", "You would pick C. That's the answer.",
+                                  "You would pick C. It could also be a human.", "Yes. The answer is C. Okay.",
+                                  "C is the correct answer."])
+def test_answer_cues_work_inside_a_spoken_reply(text):
+    assert spoken_choice(GOLD, {"text": text}) == (True, "C")
+
+
+@pytest.mark.parametrize("text", ["The answer is B. C is the correct answer.",
+                                  "They don't eat bunnies.", "Not option C."])
+def test_conflicting_answers_and_negated_mentions_are_not_correct(text):
+    assert spoken_choice(GOLD, {"text": text}) == (False, "")
+
+
+def test_question_and_answer_labels_are_not_required_by_transcription():
+    # The scorer, rather than the ASR stage, owns access to gold labels.
+    assert spoken_choice(GOLD, {"text": "bunnies", "audio_artifact": "output.wav"}) == (True, "C")

apps/aiperf_qual/tests/test_suite_audio.py

@@ -0,0 +1,49 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+"""A suite resamples dataset audio once, so the native and TRTMC sides read the same samples."""
+
+import base64
+import io
+
+import numpy as np
+import pytest
+
+from trtmc_aiperf_qual.config import ConfigError
+from trtmc_aiperf_qual.suites import _hf_rows, _wav_media
+
+soundfile = pytest.importorskip("soundfile")
+pytest.importorskip("scipy")
+
+
+def _encoded(audio, rate):
+    buffer = io.BytesIO()
+    soundfile.write(buffer, audio, rate, format="WAV", subtype="PCM_16")
+    return buffer.getvalue()
+
+
+def _decoded(media):
+    assert media["$file"]["suffix"] == ".wav"
+    return soundfile.read(io.BytesIO(base64.b64decode(media["$file"]["b64"])), dtype="float32")
+
+
+def test_audio_is_resampled_once_to_the_suite_rate():
+    tone = 0.5 * np.sin(2 * np.pi * 440 * np.arange(16_000) / 16_000)
+    audio, rate = _decoded(_wav_media(_encoded(tone, 16_000), 24_000))
+    assert rate == 24_000
+    assert audio.shape == (24_000,)
+    expected = 0.5 * np.sin(2 * np.pi * 440 * np.arange(24_000) / 24_000)
+    np.testing.assert_allclose(audio[200:-200], expected[200:-200], atol=2e-3)
+
+
+@pytest.mark.parametrize("sample_rate", [None, 16_000])
+def test_audio_keeps_its_samples_without_a_different_suite_rate(sample_rate):
+    source = (np.arange(-800, 800) * 16).astype(np.float32) / 32768  # exact PCM16 values
+    audio, rate = _decoded(_wav_media(_encoded(source, 16_000), sample_rate))
+    assert rate == 16_000
+    np.testing.assert_array_equal(audio, source)
+
+
+@pytest.mark.parametrize("sample_rate", [0, -24_000, 24_000.0, "24000", True])
+def test_invalid_suite_rates_are_rejected_before_the_dataset_loads(sample_rate):
+    with pytest.raises(ConfigError, match="audio_sample_rate"):
+        _hf_rows("dataset", object(), "revision", {"audio_sample_rate": sample_rate}, {})

apps/aiperf_qual/trtmc_aiperf_qual/absolute.py

@@ -295,6 +295,8 @@ def _suite_side(environment: Environment, service: Mapping[str, Any], model: Map
         body = json.loads(record["responses"][-1]["text"])
         key = request_sha(record["payload"]["request"])
         by_request[key] = body.get("trtmc_observation") or {}
+        if item.get("postprocess"):
+            by_request[key]["artifact_directory"] = str(Path(service["records"]).parent / "scratch" / body["id"])
         timing_by_request[key] = {"model_call_ms": float((body.get("trtmc_timing") or {}).get("model_call_ms", 0)),
                                   "completion_tokens": by_request[key].get("output_tokens")}
     observations = {index: by_request[problem["request_sha"]] for index, problem in enumerate(problems)
@@ -308,9 +310,36 @@ def _suite_side(environment: Environment, service: Mapping[str, Any], model: Map
         side["rejected"] = {"greedy": out_of_capacity}
     if failing:
         side["failed"] = {"greedy": failed_reason(failing, len(failing))}
+    if item.get("postprocess"):
+        from .services import reference_python
+
+        postprocess(environment, item, observations, out, python=reference_python(environment, model))
     return side
 
 
+def postprocess(environment: Environment, item: Mapping[str, Any], observations: dict[int, Any], out: Path, *,
+                python: str | None = None) -> None:
+    """Augment saved outputs after profiling; a separate process releases its model on exit."""
+    import subprocess
+
+    config = item["postprocess"]
+    source, target = out.parent / f"{out.name}.postprocess-input.json", out.parent / f"{out.name}.postprocess-output.json"
+    source.write_text(json.dumps(observations))
+    command = [str(environment.values.get("scoring_python") or python or environment["serve_python"]),
+               str(environment.path("repo") / config["script"]), "--input", str(source), "--output", str(target),
+               *config.get("arguments", [])]
+    with (out.parent / f"{out.name}.postprocess.log").open("w") as log:
+        subprocess.run(command, check=True, stdout=log, stderr=subprocess.STDOUT,
+                       timeout=float(config.get("timeout_s", 14400)))
+    updated = {int(index): value for index, value in json.loads(target.read_text()).items()}
+    if set(updated) != set(observations):
+        raise ValueError("output postprocessing changed the sample identities")
+    for index, original in observations.items():
+        if any(updated[index].get(key) != value for key, value in original.items()):
+            raise ValueError("output postprocessing changed original evidence")
+    observations.update(updated)
+
+
 def run_side(environment: Environment, service: Mapping[str, Any], model: Mapping[str, Any],
              item: Mapping[str, Any], problems: Sequence[Mapping[str, Any]], out: Path, *,
              capacity: bool = False) -> dict[str, Any]:
@@ -782,7 +811,8 @@ def send() -> None:  # the whole exchange, an error body included, inside the ca
 
 def keeps_artifacts(model: Mapping[str, Any]) -> bool:
     """A benchmark reads the output files the servers write (their scratch is then kept)."""
-    return any(item.get("metric") in gold_metrics.ARTIFACT_METRICS for item in model.get("absolute", []))
+    return any(item.get("postprocess") or item.get("metric") in gold_metrics.ARTIFACT_METRICS
+               for item in model.get("absolute", []))
 
 
 def run_native(environment: Environment, model: Mapping[str, Any], python: str, plans: Mapping[str, Sequence],

apps/aiperf_qual/trtmc_aiperf_qual/gold_metrics.py

@@ -41,6 +41,39 @@ def choice(gold: Any, observation: Mapping[str, Any] | None, task: str | None =
     return bool(answer) and answer == str(gold).strip().upper(), answer
 
 
+def spoken_choice(gold: Any, observation: Mapping[str, Any] | None,
+                  task: str | None = None) -> tuple[bool, str]:
+    """Score a transcribed MCQ reply by an explicit letter or unique complete option text.
+
+    Missing, ambiguous and unparsable answers are wrong. Never guess a letter.
+    """
+    text = _text(observation).strip().lower()
+    def affirmative(start: int) -> bool:
+        clause = re.split(r"[.,;:!?]", text[:start])[-1]
+        return not re.search(r"\b(?:not|never|don't|doesn't|isn't|aren't|can't|cannot|won't)\b", clause)
+
+    patterns = (
+        r"\b(?:option|choice)\s+([a-d])\b",
+        r"\b(?:the\s+)?(?:correct\s+)?answer\s+(?:is|would be)\s+([a-d])(?=\s*(?:[.,:;!?]|$))",
+        r"\b([a-d])\s+is\s+(?:the\s+)?(?:correct\s+)?answer\b",
+        r"\b(?:i|you|we)(?:'d| would)?\s+(?:say|choose|pick)\s+([a-d])(?=\s*(?:[.,:;!?]|$))",
+    )
+    letters = {match.group(1) for pattern in patterns for match in re.finditer(pattern, text)
+               if affirmative(match.start())}
+    direct = re.fullmatch(r"(?:the (?:correct )?answer is\s+)?([a-d])[.!]?", text)
+    if direct:
+        letters.add(direct.group(1))
+    if not letters:
+        for letter, option in re.findall(r"(?m)^([A-D])\.\s*(.+)$", gold["prompt"]):
+            option = " ".join(re.findall(r"\w+", option.lower()))
+            raw = re.escape(option).replace(r"\ ", r"\W+")
+            positive = any(affirmative(match.start()) for match in re.finditer(r"(?<!\w)" + raw + r"(?!\w)", text))
+            if len(option) >= 3 and positive:
+                letters.add(letter.lower())
+    answer = next(iter(letters)).upper() if len(letters) == 1 else ""
+    return bool(answer) and answer == str(gold["answer"]).strip().upper(), answer
+
+
 def _squash(text: str) -> str:
     return re.sub(r"\s+", " ", text.strip().lower())
 
@@ -206,7 +239,7 @@ def box_iou50(gold: Any, observation: Mapping[str, Any] | None, task: str | None
     return _iou(predicted, target) >= 0.5, match.group(0)
 
 
-BINARY: dict[str, Callable[..., tuple[bool, str]]] = {"choice": choice, "contains": contains, "code_pass": code_pass,
+BINARY: dict[str, Callable[..., tuple[bool, str]]] = {"choice": choice, "spoken_choice": spoken_choice, "contains": contains, "code_pass": code_pass,
                                                       "top1": top1, "box_iou50": box_iou50}
 
 

apps/aiperf_qual/trtmc_aiperf_qual/suites.py

@@ -136,7 +136,8 @@ def _aiperf_public_records(source: Mapping[str, Any], selection: Mapping[str, An
 def _hf_dataset_records(source: Mapping[str, Any], selection: Mapping[str, Any]) -> list[dict[str, Any]]:
     """Selected rows of a Hugging Face dataset at a pinned revision (``hf_dataset``): ``dataset``,
     ``subset``, ``split``, ``revision``, and the ``prompt_column`` / ``image_column`` /
-    ``audio_column`` / ``label_column`` it reads; ``columns`` copies further fields."""
+    ``audio_column`` / ``label_column`` it reads; ``columns`` copies further fields;
+    ``audio_sample_rate`` resamples the audio once for both sides."""
     from types import SimpleNamespace
 
     meta = SimpleNamespace(hf_dataset_name=require(source, "dataset", "source"), hf_subset=source.get("subset"),
@@ -254,12 +255,15 @@ def _interior_point(polygon: Any, size: Sequence[int]) -> tuple[float, float]:
 
 def _hf_rows(name: str, meta: Any, revision: str, source: Mapping[str, Any],
              selection: Mapping[str, Any]) -> list[dict[str, Any]]:
-    """Only selected rows are decoded. Audio becomes WAV (the TRTMC worker reads WAV); images keep
-    their original encoded bytes."""
+    """Only selected rows are decoded. Audio becomes WAV (the TRTMC worker reads WAV), at the source's
+    ``audio_sample_rate`` when it sets one; images keep their original encoded bytes."""
+    sample_rate = source.get("audio_sample_rate")
+    if sample_rate is not None and (isinstance(sample_rate, bool) or not isinstance(sample_rate, int)
+                                    or sample_rate <= 0):
+        raise ConfigError(f"audio_sample_rate must be a positive integer, not {sample_rate!r}")
     import io
 
     import datasets
-    import soundfile
     # Streaming datasets (for example LibriSpeech, whose config spans ~30 GB of train splits) are read
     # from the requested split only; media stay encoded until a row is selected.
     # ``data_files`` limits the download to the split's shards (for example ImageNet's validation parquet).
@@ -309,12 +313,11 @@ def _hf_rows(name: str, meta: Any, revision: str, source: Mapping[str, Any],
         if "image" in roles:
             record["image"] = _encoded_media(row[roles["image"]], ".png")
         if "audio" in roles:
-            audio, rate = soundfile.read(io.BytesIO(row[roles["audio"]]["bytes"]), dtype="float32")
-            buffer = io.BytesIO()
-            soundfile.write(buffer, audio, rate, format="WAV", subtype="PCM_16")
-            record["audio"] = {"$file": {"suffix": ".wav", "b64": base64.b64encode(buffer.getvalue()).decode()}}
+            record["audio"] = _wav_media(row[roles["audio"]]["bytes"], sample_rate)
         if label_column:
             record["label"] = row[label_column]
+        if source.get("label_fields"):
+            record["label"] = {name: row[column] for name, column in source["label_fields"].items()}
         if source.get("label_image"):
             record["label"] = {"png_b64": base64.b64encode(row[label_column]["bytes"]).decode()}
         if source.get("polygon_label") and "image" in roles:  # an object mask as a polygon, with a point inside it
@@ -347,6 +350,24 @@ def _encoded_media(value: Mapping[str, Any], default_suffix: str) -> dict[str, A
     return {"$file": {"suffix": suffix, "b64": base64.b64encode(data).decode()}}
 
 
+def _wav_media(data: bytes, sample_rate: int | None) -> dict[str, Any]:
+    """Encoded audio as a PCM16 WAV file, polyphase-resampled once to ``sample_rate`` when given: both the
+    native and the TRTMC side then read the same samples instead of resampling them differently."""
+    import numpy as np
+    import soundfile
+
+    audio, rate = soundfile.read(io.BytesIO(data), dtype="float32")
+    if sample_rate and rate != sample_rate:
+        from scipy.signal import resample_poly
+
+        divisor = math.gcd(rate, sample_rate)
+        audio = np.clip(resample_poly(audio, sample_rate // divisor, rate // divisor, axis=0), -1.0, 1.0)
+        rate = sample_rate
+    buffer = io.BytesIO()
+    soundfile.write(buffer, audio, rate, format="WAV", subtype="PCM_16")
+    return {"$file": {"suffix": ".wav", "b64": base64.b64encode(buffer.getvalue()).decode()}}
+
+
 def _field(record: Mapping[str, Any], name: str) -> Any:
     """A record field; dotted names reach into nested objects and lists (``media.0.path``)."""
     value: Any = record