feat(harbor): upload standard verifier output

examples/harbor/README.md

@@ -1,6 +1,6 @@
 # Harbor + Braintrust
 
-Runs a small, self-contained [Harbor](https://harborframework.com/) evaluation and uses Harbor's native Braintrust job plugin to sync the result. Braintrust receives a managed dataset, an experiment row for the final trial, verifier rewards, and the Harbor lifecycle and ATIF trace.
+Runs a small, self-contained [Harbor](https://harborframework.com/) evaluation and uses Harbor's native Braintrust job plugin to sync the result. Braintrust receives a managed dataset, an experiment row for the final trial, verifier rewards and standard verifier output, and the Harbor lifecycle and ATIF trace.
 
 The plugin is discovered automatically through Harbor's `braintrust` entry point. The Braintrust API key remains in the host process; it is not passed into the task container.
 
@@ -39,6 +39,10 @@ uv run harbor run \
 
 The agent solves the task in `task/`, and Harbor's verifier emits a normalized `reward` plus an `answer_length` metric. The plugin creates `jobs/braintrust-harbor-example/braintrust-sync.json` after synchronization.
 
+With the default `attachments=all` mode, the verification span and each score also include Harbor's captured `test-stdout.txt`, optional `test-stderr.txt`, and conventional `ctrf.json` output when present. The size-bounded summary and complete redacted `verifier-output.json` attachment appear together in the span output. Structured CTRF fields with sensitive key names use the plugin's standard redaction, and configured `redact_patterns` apply to both structured CTRF strings and raw verifier text.
+
+Use `attachments=structured` to keep only structured verifier evidence such as `ctrf.json`, or `attachments=none` to disable verifier and reward-detail attachments. Redaction works from key names, which raw text does not have, so **configured `redact_patterns` are the only redaction applied to raw verifier logs** — and `redact_patterns` is empty by default. Verifier logs are a common place for environment dumps, tokens in URLs, and other credentials, so configure `redact_patterns` when the evaluation environment contains sensitive values.
+
 By default, the plugin uses `Harbor` as the Braintrust project name. Override it with `--plugin-kwarg project_name=example-harbor` or through `.env`:
 
 ```dotenv

py/src/braintrust/integrations/harbor/atif.py

@@ -11,7 +11,7 @@
 from braintrust.logger import Attachment
 
 from .config import PluginConfig
-from .identity import NormalizedValue, child_span_id, normalize_json
+from .identity import NormalizedValue, child_span_id, normalize_json, try_parse_json
 
 
 _INSTRUMENTATION = "braintrust.plugin.harbor"
@@ -226,11 +226,6 @@ def _content(
                     complete = False
                     result.append(_bounded(part, config, notes, part_context).value)
                     continue
-                if len(data) > config.max_attachment_bytes:
-                    complete = False
-                    notes.add(f"{part_context}: image omitted because it exceeds max_attachment_bytes")
-                    result.append({"type": "text", "text": "[image omitted: size limit]"})
-                    continue
                 result.append(
                     {
                         "type": "image_url",
@@ -276,11 +271,14 @@ def _end_time(times: list[float], index: int, phase_end: float) -> float:
 def summarize_trajectory(trajectory_path: Path, config: PluginConfig) -> ATIFImportResult:
     """Read bounded trajectory summary data without creating detailed leaves."""
     try:
-        if trajectory_path.stat().st_size > config.max_total_attachment_bytes:
+        if trajectory_path.stat().st_size > config.max_trajectory_bytes:
             return ATIFImportResult(warnings=("trajectory omitted: size limit",))
-        trajectory = json.loads(trajectory_path.read_text())
-    except (OSError, json.JSONDecodeError) as exc:
+        data = trajectory_path.read_bytes()
+    except OSError as exc:
         return ATIFImportResult(warnings=(f"trajectory unavailable or malformed: {exc}",))
+    trajectory, parsed_ok = try_parse_json(data)
+    if not parsed_ok:
+        return ATIFImportResult(warnings=("trajectory unavailable or malformed: not valid JSON",))
     if not isinstance(trajectory, dict) or not isinstance(trajectory.get("steps"), list):
         return ATIFImportResult(warnings=("trajectory malformed: steps must be an array",))
     notes = _Notes()
@@ -325,11 +323,16 @@ def import_trajectory(
         trajectory = _trajectory_data
     else:
         try:
-            if trajectory_path.stat().st_size > config.max_total_attachment_bytes:
+            # Unlike an attachment, this document is parsed into the host process
+            # rather than handed to object storage, so its size is bounded.
+            if trajectory_path.stat().st_size > config.max_trajectory_bytes:
                 return ATIFImportResult(warnings=("trajectory omitted: size limit",))
-            trajectory = json.loads(trajectory_path.read_text())
-        except (OSError, json.JSONDecodeError) as exc:
+            data = trajectory_path.read_bytes()
+        except OSError as exc:
             return ATIFImportResult(warnings=(f"trajectory unavailable or malformed: {exc}",))
+        trajectory, parsed_ok = try_parse_json(data)
+        if not parsed_ok:
+            return ATIFImportResult(warnings=("trajectory unavailable or malformed: not valid JSON",))
     if not isinstance(trajectory, dict) or not isinstance(trajectory.get("steps"), list):
         return ATIFImportResult(warnings=("trajectory malformed: steps must be an array",))
 

py/src/braintrust/integrations/harbor/compat.py

@@ -194,6 +194,10 @@ def reward_details_paths(result: Any) -> list[tuple[str | None, Path]]:
     return _step_paths(result, "verifier", "reward-details.json")
 
 
+def verifier_output_paths(result: Any) -> list[tuple[str | None, Path]]:
+    return _step_paths(result, "verifier")
+
+
 def artifact_manifest_paths(result: Any) -> list[tuple[str | None, Path]]:
     return _step_paths(result, "artifacts", "manifest.json")
 

py/src/braintrust/integrations/harbor/config.py

@@ -98,11 +98,10 @@ class PluginConfig:
     classifier_rules: dict[str, str] = field(default_factory=dict)
     invalid_score_policy: str = "metric"
     include_tracebacks: bool = False
-    attachments: str = "verifier-details"
+    attachments: str = "all"
     artifact_include: tuple[str, ...] = ()
-    max_attachment_bytes: int = 5_000_000
-    max_total_attachment_bytes: int = 20_000_000
     max_content_bytes: int = 20_000
+    max_trajectory_bytes: int = 20_000_000
     log_retry_attempts: bool = False
     strict: bool = False
     redact_patterns: tuple[str, ...] = ()
@@ -124,9 +123,8 @@ def from_options(cls, **options: Any) -> "PluginConfig":
             values[name] = _parse_bool(values[name], name)
         for name in (
             "max_custom_metadata_bytes",
-            "max_attachment_bytes",
-            "max_total_attachment_bytes",
             "max_content_bytes",
+            "max_trajectory_bytes",
         ):
             values[name] = _parse_int(values[name], name)
         for name in ("score_keys", "metric_keys", "artifact_include", "redact_patterns"):
@@ -157,12 +155,10 @@ def validate(self) -> None:
             raise ValueError("log_retry_attempts=True is not implemented; only the final attempt is logged")
         if self.invalid_score_policy not in {"metric", "drop", "error"}:
             raise ValueError("invalid_score_policy must be 'metric', 'drop', or 'error'")
-        if self.attachments not in {"none", "verifier-details", "all"}:
-            raise ValueError("attachments must be 'none', 'verifier-details', or 'all'")
+        if self.attachments not in {"none", "structured", "all"}:
+            raise ValueError("attachments must be 'none', 'structured', or 'all'")
         if self.artifact_include and self.attachments != "all":
             raise ValueError("artifact_include requires attachments='all'")
-        if self.max_total_attachment_bytes < self.max_attachment_bytes:
-            raise ValueError("max_total_attachment_bytes must be at least max_attachment_bytes")
 
         for score_pattern in self.score_keys:
             for metric_pattern in self.metric_keys:

py/src/braintrust/integrations/harbor/identity.py

@@ -108,6 +108,18 @@ def _is_secret_key(key: str, value: Any) -> bool:
     return _key_segments(key).isdisjoint(_COUNTER_SEGMENTS)
 
 
+def try_parse_json(data: bytes) -> tuple[Any, bool]:
+    """Parse a task-controlled document, reporting failure rather than raising.
+
+    json.loads raises RecursionError, not JSONDecodeError, for a deeply nested
+    document, and a task can write one to any file this package reads.
+    """
+    try:
+        return json.loads(data), True
+    except (UnicodeDecodeError, json.JSONDecodeError, RecursionError):
+        return None, False
+
+
 def _json_size(value: Any) -> int:
     try:
         return len(canonical_json(value).encode("utf-8"))
@@ -118,7 +130,7 @@ def _json_size(value: Any) -> int:
 def normalize_json(
     value: Any,
     *,
-    max_bytes: int,
+    max_bytes: int | None,
     redact_patterns: tuple[str, ...] = (),
     max_depth: int = 8,
     redact_absolute_paths: bool = True,
@@ -128,6 +140,9 @@ def normalize_json(
     Set ``redact_absolute_paths=False`` for payloads produced inside the task
     sandbox: their absolute paths are container paths the agent actually operated
     on, so redacting them erases the substance of filesystem tool calls.
+
+    ``max_bytes=None`` redacts without truncating, for payloads bound for an
+    attachment rather than a span field.
     """
     warnings: list[str] = []
     compiled_patterns = tuple(re.compile(pattern) for pattern in redact_patterns)
@@ -176,7 +191,7 @@ def walk(item: Any, path: str, depth: int, key: str | None = None) -> Any:
         return f"[DROPPED: {type(item).__name__}]"
 
     normalized = walk(value, "", 0)
-    if _json_size(normalized) <= max_bytes:
+    if max_bytes is None or _json_size(normalized) <= max_bytes:
         return NormalizedValue(normalized, tuple(warnings))
 
     # Fitting a container to a byte budget needs each entry's serialized size, not