From ea06160e954484119b47d51ab995866a225ce284 Mon Sep 17 00:00:00 2001 From: pearseona Date: Sun, 16 Aug 2026 16:13:36 +0900 Subject: [PATCH 1/5] test: add full-dataset evaluatino metrics --- .../SMSModel/hybrid_evaluation/__init__.py | 4 + .../SMSModel/hybrid_evaluation/metrics.py | 82 +++++++++++++++++++ .../SMSModel/hybrid_evaluation/models.py | 23 ++++++ .../hybrid_evaluation/test_metrics.py | 81 ++++++++++++++++++ 4 files changed, 190 insertions(+) diff --git a/data_science/SMSModel/hybrid_evaluation/__init__.py b/data_science/SMSModel/hybrid_evaluation/__init__.py index 0375144..1a81790 100644 --- a/data_science/SMSModel/hybrid_evaluation/__init__.py +++ b/data_science/SMSModel/hybrid_evaluation/__init__.py @@ -4,6 +4,7 @@ calculate_classification_metrics, calculate_cost_metrics, calculate_cost_reduction_rate, + calculate_full_dataset_metrics, calculate_latency_metrics, calculate_operational_metrics, ) @@ -12,6 +13,7 @@ CostMetrics, EvaluationMode, EvaluationRecord, + FullDatasetMetrics, LatencyMetrics, OperationalMetrics, OperationalOutcome, @@ -41,6 +43,7 @@ "EvaluationMode", "EvaluationRecord", "EvaluationSample", + "FullDatasetMetrics", "HybridEvaluationRunner", "LatencyMetrics", "OperationalMetrics", @@ -52,6 +55,7 @@ "calculate_classification_metrics", "calculate_cost_metrics", "calculate_cost_reduction_rate", + "calculate_full_dataset_metrics", "calculate_dataset_fingerprint", "calculate_latency_metrics", "calculate_operational_metrics", diff --git a/data_science/SMSModel/hybrid_evaluation/metrics.py b/data_science/SMSModel/hybrid_evaluation/metrics.py index 7eb56ab..9f3d24e 100644 --- a/data_science/SMSModel/hybrid_evaluation/metrics.py +++ b/data_science/SMSModel/hybrid_evaluation/metrics.py @@ -17,6 +17,7 @@ from data_science.SMSModel.hybrid_evaluation.models import ( ClassificationMetrics, CostMetrics, + FullDatasetMetrics, LatencyMetrics, OperationalMetrics, OperationalOutcome, @@ -24,6 +25,7 @@ ) LABEL_ORDER = ("normal", "phishing") +UNAVAILABLE_LABEL = "unknown" def _validate_binary_labels( y_true: Sequence[str], @@ -134,6 +136,86 @@ def calculate_classification_metrics( true_positive=true_positive, ) + +def calculate_full_dataset_metrics( + y_true: Sequence[str], + y_pred: Sequence[str], +) -> FullDatasetMetrics: + """UNKNOWN을 실패로 포함한 전체 데이터셋 지표를 계산""" + + true_values = np.asarray(y_true, dtype=str) + predicted_values = np.asarray(y_pred, dtype=str) + + if true_values.ndim != 1 or predicted_values.ndim != 1: + raise ValueError( + "y_true and y_pred must be one-dimensional" + ) + + if len(true_values) == 0: + raise ValueError( + "cannot calculate full-dataset metrics from empty labels" + ) + + if len(true_values) != len(predicted_values): + raise ValueError( + "y_true and y_pred must have the same length" + ) + + binary_labels = set(LABEL_ORDER) + + if not set(true_values).issubset(binary_labels): + raise ValueError( + "y_true contains unsupported labels" + ) + + allowed_predictions = {*binary_labels, UNAVAILABLE_LABEL} + + if not set(predicted_values).issubset(allowed_predictions): + raise ValueError( + "y_pred contains unsupported labels" + ) + + available_mask = np.isin(predicted_values, LABEL_ORDER) + unavailable_mask = predicted_values == UNAVAILABLE_LABEL + correct_mask = available_mask & (true_values == predicted_values) + incorrect_mask = available_mask & (true_values != predicted_values) + phishing_mask = true_values == "phishing" + detected_phishing_mask = phishing_mask & ( + predicted_values == "phishing" + ) + + total_sample_count = len(true_values) + available_count = int(available_mask.sum()) + unavailable_count = int(unavailable_mask.sum()) + correct_count = int(correct_mask.sum()) + incorrect_count = int(incorrect_mask.sum()) + actual_normal_count = int((true_values == "normal").sum()) + actual_phishing_count = int(phishing_mask.sum()) + detected_phishing_count = int(detected_phishing_mask.sum()) + missed_phishing_count = ( + actual_phishing_count - detected_phishing_count + ) + + phishing_detection_rate = ( + detected_phishing_count / actual_phishing_count + if actual_phishing_count > 0 + else 0.0 + ) + + return FullDatasetMetrics( + total_sample_count=total_sample_count, + available_count=available_count, + unavailable_count=unavailable_count, + correct_count=correct_count, + incorrect_count=incorrect_count, + accuracy=correct_count / total_sample_count, + actual_normal_count=actual_normal_count, + actual_phishing_count=actual_phishing_count, + detected_phishing_count=detected_phishing_count, + missed_phishing_count=missed_phishing_count, + phishing_detection_rate=phishing_detection_rate, + ) + def calculate_latency_metrics( durations_ms: Sequence[float], ) -> LatencyMetrics: diff --git a/data_science/SMSModel/hybrid_evaluation/models.py b/data_science/SMSModel/hybrid_evaluation/models.py index 91d8d4a..93faf8e 100644 --- a/data_science/SMSModel/hybrid_evaluation/models.py +++ b/data_science/SMSModel/hybrid_evaluation/models.py @@ -172,3 +172,26 @@ def to_dict(self) -> dict[str, Any]: payload["mode"] = self.mode.value return payload + + +@dataclass(frozen=True) +class FullDatasetMetrics: + """전체 데이터셋의 분류 및 피싱 탐지 성능 지표""" + + total_sample_count: int + available_count: int + unavailable_count: int + correct_count: int + incorrect_count: int + accuracy: float + + actual_normal_count: int + actual_phishing_count: int + detected_phishing_count: int + missed_phishing_count: int + phishing_detection_rate: float + + def to_dict(self) -> dict[str, Any]: + """JSON 직렬화가 가능한 dictionary로 변환""" + + return asdict(self) diff --git a/tests/data_science/SMSModel/hybrid_evaluation/test_metrics.py b/tests/data_science/SMSModel/hybrid_evaluation/test_metrics.py index 972ef08..b2a8339 100644 --- a/tests/data_science/SMSModel/hybrid_evaluation/test_metrics.py +++ b/tests/data_science/SMSModel/hybrid_evaluation/test_metrics.py @@ -10,6 +10,7 @@ calculate_classification_metrics, calculate_cost_metrics, calculate_cost_reduction_rate, + calculate_full_dataset_metrics, calculate_latency_metrics, calculate_operational_metrics, ) @@ -100,6 +101,86 @@ def test_classification_metrics_handle_no_positive_prediction() -> None: assert metrics.false_negative == 1 +def test_full_dataset_metrics_match_binary_metrics_when_all_available() -> None: + """모든 결과가 있으면 전체 정확도는 이진 분류 정확도와 같습니다.""" + + metrics = calculate_full_dataset_metrics( + y_true=["normal", "normal", "phishing", "phishing"], + y_pred=["normal", "phishing", "normal", "phishing"], + ) + + assert metrics.total_sample_count == 4 + assert metrics.available_count == 4 + assert metrics.unavailable_count == 0 + assert metrics.correct_count == 2 + assert metrics.incorrect_count == 2 + assert metrics.accuracy == pytest.approx(0.5) + assert metrics.actual_normal_count == 2 + assert metrics.actual_phishing_count == 2 + assert metrics.detected_phishing_count == 1 + assert metrics.missed_phishing_count == 1 + assert metrics.phishing_detection_rate == pytest.approx(0.5) + + +def test_full_dataset_metrics_count_unknown_as_unavailable_failure() -> None: + """UNKNOWN은 전체 분모와 피싱 미탐에 포함합니다.""" + + metrics = calculate_full_dataset_metrics( + y_true=["normal", "phishing", "phishing"], + y_pred=["unknown", "unknown", "phishing"], + ) + + assert metrics.total_sample_count == 3 + assert metrics.available_count == 1 + assert metrics.unavailable_count == 2 + assert metrics.correct_count == 1 + assert metrics.incorrect_count == 0 + assert metrics.accuracy == pytest.approx(1 / 3) + assert metrics.detected_phishing_count == 1 + assert metrics.missed_phishing_count == 1 + assert metrics.phishing_detection_rate == pytest.approx(0.5) + assert ( + metrics.correct_count + + metrics.incorrect_count + + metrics.unavailable_count + == metrics.total_sample_count + ) + + +def test_full_dataset_metrics_handle_no_phishing_samples() -> None: + """피싱 표본이 없으면 탐지율을 0으로 반환합니다.""" + + metrics = calculate_full_dataset_metrics( + y_true=["normal", "normal"], + y_pred=["normal", "unknown"], + ) + + assert metrics.actual_phishing_count == 0 + assert metrics.detected_phishing_count == 0 + assert metrics.missed_phishing_count == 0 + assert metrics.phishing_detection_rate == 0.0 + + +@pytest.mark.parametrize( + ("y_true", "y_pred", "error_message"), + [ + ([], [], "empty labels"), + (["normal"], ["normal", "unknown"], "same length"), + (["safe"], ["normal"], "y_true contains unsupported labels"), + (["normal"], ["unavailable"], "y_pred contains unsupported labels"), + ], +) +def test_rejects_invalid_full_dataset_inputs( + y_true: list[str], + y_pred: list[str], + error_message: str, +) -> None: + """전체 지표도 빈 입력, 길이 및 label 계약을 검증합니다.""" + + with pytest.raises(ValueError, match=error_message): + calculate_full_dataset_metrics(y_true, y_pred) + + @pytest.mark.parametrize( ("y_true", "y_pred", "error_message"), [ From ac2bfa367c2a24c056d401c3324b0116941bcddd Mon Sep 17 00:00:00 2001 From: pearseona Date: Sun, 16 Aug 2026 16:31:56 +0900 Subject: [PATCH 2/5] test: separate avaliable-only and full-dataset results --- .../SMSModel/hybrid_evaluation/reporting.py | 97 +++++++++++++++---- .../hybrid_evaluation/test_reporting.py | 58 ++++++++++- 2 files changed, 134 insertions(+), 21 deletions(-) diff --git a/data_science/SMSModel/hybrid_evaluation/reporting.py b/data_science/SMSModel/hybrid_evaluation/reporting.py index 557e592..ad55a56 100644 --- a/data_science/SMSModel/hybrid_evaluation/reporting.py +++ b/data_science/SMSModel/hybrid_evaluation/reporting.py @@ -11,12 +11,13 @@ from .metrics import ( calculate_classification_metrics, calculate_cost_metrics, + calculate_full_dataset_metrics, calculate_latency_metrics, calculate_operational_metrics, ) from .models import EvaluationMode, OperationalOutcome, TokenUsage -REPORT_SCHEMA_VERSION = 2 +REPORT_SCHEMA_VERSION = 3 _MODES = tuple(EvaluationMode) _BINARY_LABELS = {"normal", "phishing"} @@ -67,6 +68,13 @@ def build_comparison_report( llm_report = mode_reports[EvaluationMode.LLM_ONLY.value] hybrid_report = mode_reports[EvaluationMode.HYBRID.value] + reference_records = grouped[EvaluationMode.SELF_MODEL_ONLY] + normal_count = sum( + record["expected_label"] == "normal" + for record in reference_records + ) + phishing_count = len(reference_records) - normal_count + comparisons = { "llm_call_reduction_rate": _reduction_rate( llm_report["operations"]["llm_call_rate"], @@ -85,15 +93,25 @@ def build_comparison_report( hybrid_report["latency_ms"]["p95_ms"], ), "hybrid_recall_delta_vs_self_model": _metric_delta( - hybrid_report["classification"], - self_report["classification"], + hybrid_report["classification"]["available_only"], + self_report["classification"]["available_only"], "recall", ), "hybrid_f2_delta_vs_self_model": _metric_delta( - hybrid_report["classification"], - self_report["classification"], + hybrid_report["classification"]["available_only"], + self_report["classification"]["available_only"], "f2", ), + "hybrid_full_accuracy_delta_vs_self_model": _metric_delta( + hybrid_report["classification"]["full_dataset"], + self_report["classification"]["full_dataset"], + "accuracy", + ), + "hybrid_detection_rate_delta_vs_self_model": _metric_delta( + hybrid_report["classification"]["full_dataset"], + self_report["classification"]["full_dataset"], + "phishing_detection_rate", + ), } target_assessment = _build_target_assessment( @@ -110,6 +128,12 @@ def build_comparison_report( "sample_count": len(sample_ids), "dataset_fingerprint": metadata["dataset_fingerprint"], "evaluation_schema_version": metadata["evaluation_schema_version"], + "dataset": { + "total_count": len(sample_ids), + "normal_count": normal_count, + "phishing_count": phishing_count, + "positive_label": "phishing", + }, "stacking_artifact": stacking, "model": { "provider": "AWS_BEDROCK", @@ -146,9 +170,19 @@ def build_comparison_report( }, "classification_policy": { "positive_label": "phishing", + "available_only_metrics": ( + "Binary classification metrics use only records with an " + "available normal or phishing prediction." + ), + "full_dataset_metrics": ( + "Full-dataset accuracy uses every test sample; unavailable " + "results count as unsuccessful outcomes. Phishing detection " + "rate counts unavailable phishing samples as missed." + ), "unknown_handling": ( - "UNKNOWN results are excluded from binary classification " - "metrics and reported as unavailable_count." + "UNKNOWN results are excluded from available-only binary " + "metrics, retained in full-dataset denominators, and reported " + "as unavailable_count." ), }, } @@ -202,8 +236,13 @@ def render_markdown_report(report: dict[str, Any]) -> str: } ordered = [modes[mode.value] for mode in _MODES] - def add_row(label: str, path: tuple[str, str], formatter) -> None: - values = [mode[path[0]][path[1]] for mode in ordered] + def add_row(label: str, path: tuple[str, ...], formatter) -> None: + values = [] + for mode in ordered: + value: Any = mode + for key in path: + value = value.get(key) if value is not None else None + values.append(value) lines.append( f"| {label} | " + " | ".join(formatter(value) for value in values) + " |" ) @@ -215,7 +254,11 @@ def add_row(label: str, path: tuple[str, str], formatter) -> None: ("f1", "F1"), ("f2", "F2"), ): - add_row(label, ("classification", metric_name), _format_ratio) + add_row( + label, + ("classification", "available_only", metric_name), + _format_ratio, + ) add_row("평균 지연시간 (ms)", ("latency_ms", "average_ms"), _format_number) add_row("P50 지연시간 (ms)", ("latency_ms", "p50_ms"), _format_number) @@ -343,7 +386,7 @@ def render_csv_report(report: dict[str, Any]) -> str: writer.writeheader() for mode in _MODES: summary = report["modes"][mode.value] - classification = summary["classification"] or {} + classification = summary["classification"]["available_only"] or {} writer.writerow({ "generated_at": report["generated_at"], "source_split": report["source_split"], @@ -453,7 +496,7 @@ def _summarize_mode( if record["result_available"] is True and record["predicted_label"] in _BINARY_LABELS ] - classification = ( + available_only = ( calculate_classification_metrics( [str(record["expected_label"]) for record in available], [str(record["predicted_label"]) for record in available], @@ -461,6 +504,17 @@ def _summarize_mode( if available else None ) + full_predictions = [ + str(record["predicted_label"]) + if record["result_available"] is True + and record["predicted_label"] in _BINARY_LABELS + else "unknown" + for record in records + ] + full_dataset = calculate_full_dataset_metrics( + [str(record["expected_label"]) for record in records], + full_predictions, + ).to_dict() operations = calculate_operational_metrics( [ OperationalOutcome( @@ -487,7 +541,10 @@ def _summarize_mode( output_price_per_million=output_price_per_million, ).to_dict() return { - "classification": classification, + "classification": { + "available_only": available_only, + "full_dataset": full_dataset, + }, "availability": { "available_count": len(available), "unavailable_count": len(records) - len(available), @@ -567,13 +624,19 @@ def _build_target_assessment( comparisons: dict[str, float | None], target_recall: float, ) -> dict[str, Any]: - hybrid_classification = hybrid_report["classification"] - self_classification = self_report["classification"] - if hybrid_classification is None or self_classification is None: + hybrid_available = hybrid_report["classification"]["available_only"] + self_available = self_report["classification"]["available_only"] + hybrid_full = hybrid_report["classification"]["full_dataset"] + if hybrid_available is None or self_available is None: raise ValueError("target assessment requires classification metrics") specifications = [ - ("Hybrid Recall", f">= {target_recall:.4f}", hybrid_classification["recall"], hybrid_classification["recall"] >= target_recall), + ( + "Hybrid full-dataset phishing detection rate", + f">= {target_recall:.4f}", + hybrid_full["phishing_detection_rate"], + hybrid_full["phishing_detection_rate"] >= target_recall, + ), ("Hybrid F2 vs Stacking-only", ">= 0 delta", comparisons["hybrid_f2_delta_vs_self_model"], (comparisons["hybrid_f2_delta_vs_self_model"] or 0) >= 0), ("Claude call reduction", "> 0%", comparisons["llm_call_reduction_rate"], (comparisons["llm_call_reduction_rate"] or 0) > 0), ("Cost reduction", "> 0%", comparisons["cost_per_message_reduction_rate"], (comparisons["cost_per_message_reduction_rate"] or 0) > 0), diff --git a/tests/data_science/SMSModel/hybrid_evaluation/test_reporting.py b/tests/data_science/SMSModel/hybrid_evaluation/test_reporting.py index 31a5dc2..abc668e 100644 --- a/tests/data_science/SMSModel/hybrid_evaluation/test_reporting.py +++ b/tests/data_science/SMSModel/hybrid_evaluation/test_reporting.py @@ -112,9 +112,32 @@ def test_builds_three_mode_comparison_and_reductions() -> None: modes = report["modes"] assert report["sample_count"] == 2 - assert modes["SELF_MODEL_ONLY"]["classification"]["recall"] == 0.0 - assert modes["LLM_ONLY"]["classification"]["recall"] == 1.0 - assert modes["HYBRID"]["classification"]["f2"] == 1.0 + assert report["report_schema_version"] == 3 + assert report["dataset"] == { + "total_count": 2, + "normal_count": 1, + "phishing_count": 1, + "positive_label": "phishing", + } + assert "every test sample" in report["classification_policy"][ + "full_dataset_metrics" + ] + assert ( + modes["SELF_MODEL_ONLY"]["classification"]["available_only"]["recall"] + == 0.0 + ) + assert ( + modes["LLM_ONLY"]["classification"]["available_only"]["recall"] + == 1.0 + ) + assert ( + modes["HYBRID"]["classification"]["available_only"]["f2"] + == 1.0 + ) + assert ( + modes["HYBRID"]["classification"]["full_dataset"]["accuracy"] + == 1.0 + ) assert modes["SELF_MODEL_ONLY"]["operations"]["llm_call_rate"] == 0.0 assert modes["LLM_ONLY"]["operations"]["llm_call_rate"] == 1.0 assert modes["HYBRID"]["operations"]["llm_call_rate"] == 0.5 @@ -145,7 +168,34 @@ def test_unknown_is_excluded_and_reported_as_unavailable() -> None: hybrid = report["modes"]["HYBRID"] assert hybrid["availability"]["unavailable_count"] == 1 - assert hybrid["classification"]["sample_count"] == 1 + assert hybrid["classification"]["available_only"]["sample_count"] == 1 + assert hybrid["classification"]["full_dataset"]["total_sample_count"] == 2 + assert hybrid["classification"]["full_dataset"]["unavailable_count"] == 1 + assert hybrid["classification"]["full_dataset"]["accuracy"] == 0.5 + assert ( + hybrid["classification"]["full_dataset"]["phishing_detection_rate"] + == 0.0 + ) + + +def test_full_dataset_metrics_use_all_records_in_each_mode() -> None: + records = _records() + for record in records: + if record["mode"] == "LLM_ONLY" and record["sample_id"] == "a": + record.update( + predicted_label="unknown", + result_available=False, + llm_available=False, + all_engines_unavailable=True, + ) + + report = _build(records) + llm = report["modes"]["LLM_ONLY"] + + assert llm["classification"]["available_only"]["accuracy"] == 1.0 + assert llm["classification"]["full_dataset"]["accuracy"] == 0.5 + assert llm["classification"]["full_dataset"]["correct_count"] == 1 + assert llm["classification"]["full_dataset"]["unavailable_count"] == 1 def test_rejects_mismatched_sample_sets() -> None: From bc6161b8b9844c884edd724f2c2a5dc2b7f088d8 Mon Sep 17 00:00:00 2001 From: pearseona Date: Sun, 16 Aug 2026 16:36:11 +0900 Subject: [PATCH 3/5] test: record reproducible evaluation provenance --- .../SMSModel/hybrid_evaluation/reporting.py | 76 ++++++++++++++- .../SMSModel/run_hybrid_evaluation.py | 28 ++++++ .../hybrid_evaluation/test_reporting.py | 53 ++++++++++- .../SMSModel/test_run_hybrid_evaluation.py | 94 +++++++++++++++++++ 4 files changed, 249 insertions(+), 2 deletions(-) create mode 100644 tests/data_science/SMSModel/test_run_hybrid_evaluation.py diff --git a/data_science/SMSModel/hybrid_evaluation/reporting.py b/data_science/SMSModel/hybrid_evaluation/reporting.py index ad55a56..ddc8cf8 100644 --- a/data_science/SMSModel/hybrid_evaluation/reporting.py +++ b/data_science/SMSModel/hybrid_evaluation/reporting.py @@ -128,6 +128,12 @@ def build_comparison_report( "sample_count": len(sample_ids), "dataset_fingerprint": metadata["dataset_fingerprint"], "evaluation_schema_version": metadata["evaluation_schema_version"], + "evaluation_provenance": { + "split_manifest": metadata["split_manifest"], + "split_manifest_sha256": metadata["split_manifest_sha256"], + "random_state": metadata["random_state"], + "positive_label": metadata["positive_label"], + }, "dataset": { "total_count": len(sample_ids), "normal_count": normal_count, @@ -588,6 +594,7 @@ def _validate_source_metadata(metadata: dict[str, Any]) -> dict[str, Any]: raise ValueError("evaluation records must come from test split") for field in ( "dataset_fingerprint", + "split_manifest", "model_id", "region", "prompt_version", @@ -596,12 +603,33 @@ def _validate_source_metadata(metadata: dict[str, Any]) -> dict[str, Any]: version = metadata.get("evaluation_schema_version") if not isinstance(version, int) or isinstance(version, bool) or version <= 0: raise ValueError("evaluation_schema_version is invalid") + + manifest_sha256 = _non_empty_string( + metadata.get("split_manifest_sha256"), + "split_manifest_sha256", + ) + if not _is_sha256(manifest_sha256): + raise ValueError( + "split_manifest_sha256 must be a hexadecimal SHA-256 digest" + ) + + random_state = metadata.get("random_state") + if ( + not isinstance(random_state, int) + or isinstance(random_state, bool) + or random_state < 0 + ): + raise ValueError("random_state must be a non-negative integer") + + if metadata.get("positive_label") != "phishing": + raise ValueError("positive_label must be phishing") + return metadata def _validate_stacking_metadata(metadata: dict[str, Any]) -> dict[str, Any]: sha256 = _non_empty_string(metadata.get("model_sha256"), "model_sha256") - if len(sha256) != 64 or any(character not in "0123456789abcdef" for character in sha256.lower()): + if not _is_sha256(sha256): raise ValueError("model_sha256 must be a hexadecimal SHA-256 digest") version = metadata.get("schema_version") if not isinstance(version, int) or isinstance(version, bool) or version <= 0: @@ -609,11 +637,50 @@ def _validate_stacking_metadata(metadata: dict[str, Any]) -> dict[str, Any]: model = metadata.get("model") if not isinstance(model, dict): raise ValueError("stacking model metadata is missing") + + random_state = model.get("random_state") + if ( + not isinstance(random_state, int) + or isinstance(random_state, bool) + or random_state < 0 + ): + raise ValueError( + "stacking random_state must be a non-negative integer" + ) + + threshold = model.get("threshold") + if ( + isinstance(threshold, bool) + or not isinstance(threshold, (int, float)) + or not math.isfinite(float(threshold)) + or not 0 <= float(threshold) <= 1 + ): + raise ValueError("stacking threshold is invalid") + + validation = metadata.get("validation") + if not isinstance(validation, dict): + raise ValueError("stacking validation metadata is missing") + target_recall = validation.get("target_recall") + if ( + isinstance(target_recall, bool) + or not isinstance(target_recall, (int, float)) + or not math.isfinite(float(target_recall)) + or not 0 < float(target_recall) <= 1 + ): + raise ValueError("stacking validation target_recall is invalid") + return { "sha256": sha256, "schema_version": version, "model_name": _non_empty_string(model.get("model_name"), "model_name"), "created_at": _non_empty_string(metadata.get("created_at"), "created_at"), + "random_state": random_state, + "classification_threshold": float(threshold), + "threshold_source_split": "validation", + "threshold_selection_metric": ( + "maximize_f2_subject_to_target_recall" + ), + "target_recall": float(target_recall), } @@ -682,6 +749,13 @@ def _non_empty_string(value: object, field: str) -> str: return value.strip() +def _is_sha256(value: str) -> bool: + return len(value) == 64 and all( + character in "0123456789abcdef" + for character in value.lower() + ) + + def _format_ratio(value: float | None) -> str: return "N/A" if value is None else f"{value:.4f}" diff --git a/data_science/SMSModel/run_hybrid_evaluation.py b/data_science/SMSModel/run_hybrid_evaluation.py index fbb9126..68312e3 100644 --- a/data_science/SMSModel/run_hybrid_evaluation.py +++ b/data_science/SMSModel/run_hybrid_evaluation.py @@ -3,6 +3,7 @@ import argparse import asyncio +import hashlib import json import math from pathlib import Path @@ -30,6 +31,7 @@ from data_science.SMSModel.train_sms import ( DATA_PATH, SPLIT_MANIFEST_PATH, + build_dataset_split_config, load_data, split_data, ) @@ -152,6 +154,18 @@ def calculate_test_dataset_fingerprint(test) -> str: for row in test.itertuples(index=False) ) + +def calculate_file_sha256(path: Path) -> str: + """평가 입력 파일의 재현성 확인용 SHA-256을 계산""" + + digest = hashlib.sha256() + + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + + return digest.hexdigest() + def load_frozen_validation_policy() -> ConditionalLlmPolicy: """validation에서 선정된 임계값만 읽음""" @@ -396,6 +410,14 @@ def save_evaluation_records( dataset_fingerprint: str, ) -> None: """원문 없는 평가 레코드를 저장""" + + if not SPLIT_MANIFEST_PATH.is_file(): + raise FileNotFoundError( + f"split manifest is required: {SPLIT_MANIFEST_PATH}" + ) + + split_config = build_dataset_split_config() + _atomic_write_json( EVALUATION_RECORDS_PATH, { @@ -404,6 +426,12 @@ def save_evaluation_records( ), "source_split": "test", "dataset_fingerprint": dataset_fingerprint, + "split_manifest": SPLIT_MANIFEST_PATH.name, + "split_manifest_sha256": calculate_file_sha256( + SPLIT_MANIFEST_PATH + ), + "random_state": split_config.random_state, + "positive_label": "phishing", "model_id": settings.BEDROCK_MODEL_ID, "region": settings.AWS_REGION, "prompt_version": build_prompt_version(), diff --git a/tests/data_science/SMSModel/hybrid_evaluation/test_reporting.py b/tests/data_science/SMSModel/hybrid_evaluation/test_reporting.py index abc668e..03197f1 100644 --- a/tests/data_science/SMSModel/hybrid_evaluation/test_reporting.py +++ b/tests/data_science/SMSModel/hybrid_evaluation/test_reporting.py @@ -65,6 +65,10 @@ def _metadata() -> dict: return { "source_split": "test", "dataset_fingerprint": "dataset", + "split_manifest": "sms_split_v1.csv", + "split_manifest_sha256": "b" * 64, + "random_state": 42, + "positive_label": "phishing", "evaluation_schema_version": 1, "model_id": "test-model", "region": "us-east-1", @@ -89,7 +93,14 @@ def _stacking_metadata() -> dict: "created_at": "2026-08-11T00:00:00+00:00", "model_sha256": "a" * 64, "schema_version": 1, - "model": {"model_name": "stacking_phishing_classifier"}, + "model": { + "model_name": "stacking_phishing_classifier", + "random_state": 42, + "threshold": 0.25, + }, + "validation": { + "target_recall": 0.95, + }, } @@ -122,6 +133,17 @@ def test_builds_three_mode_comparison_and_reductions() -> None: assert "every test sample" in report["classification_policy"][ "full_dataset_metrics" ] + assert report["evaluation_provenance"] == { + "split_manifest": "sms_split_v1.csv", + "split_manifest_sha256": "b" * 64, + "random_state": 42, + "positive_label": "phishing", + } + assert report["stacking_artifact"]["classification_threshold"] == 0.25 + assert report["stacking_artifact"]["threshold_source_split"] == "validation" + assert report["stacking_artifact"]["threshold_selection_metric"] == ( + "maximize_f2_subject_to_target_recall" + ) assert ( modes["SELF_MODEL_ONLY"]["classification"]["available_only"]["recall"] == 0.0 @@ -245,6 +267,35 @@ def test_rejects_invalid_price() -> None: ) +@pytest.mark.parametrize( + ("field", "value", "error_message"), + [ + ("split_manifest_sha256", "invalid", "SHA-256"), + ("random_state", -1, "random_state"), + ("positive_label", "normal", "positive_label"), + ], +) +def test_rejects_invalid_evaluation_provenance( + field: str, + value, + error_message: str, +) -> None: + metadata = _metadata() + metadata[field] = value + + with pytest.raises(ValueError, match=error_message): + build_comparison_report( + _records(), + source_metadata=metadata, + policy=_policy(), + input_price_per_million=1.0, + output_price_per_million=5.0, + currency="USD", + pricing_as_of="2026-08-13", + stacking_metadata=_stacking_metadata(), + ) + + def test_renders_markdown_without_sample_identifiers() -> None: markdown = render_markdown_report(_build()) diff --git a/tests/data_science/SMSModel/test_run_hybrid_evaluation.py b/tests/data_science/SMSModel/test_run_hybrid_evaluation.py new file mode 100644 index 0000000..fa45f9a --- /dev/null +++ b/tests/data_science/SMSModel/test_run_hybrid_evaluation.py @@ -0,0 +1,94 @@ +"""하이브리드 평가 실행 결과의 provenance 저장 테스트.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +from data_science.SMSModel import run_hybrid_evaluation as evaluation +from data_science.SMSModel.hybrid_evaluation import ( + EvaluationMode, + EvaluationRecord, +) + + +def _record() -> EvaluationRecord: + return EvaluationRecord( + sample_id="sha256:test", + mode=EvaluationMode.SELF_MODEL_ONLY, + expected_label="phishing", + predicted_label="phishing", + latency_ms=1.0, + result_available=True, + llm_called=False, + llm_available=False, + fallback_applied=False, + all_engines_unavailable=False, + decision_source="STACKING", + routing_decision=None, + routing_reason=None, + error_code=None, + llm_provider=None, + llm_model=None, + input_tokens=None, + output_tokens=None, + ) + + +def test_saves_reproducible_evaluation_provenance( + monkeypatch, + tmp_path: Path, +) -> None: + manifest_path = tmp_path / "sms_split_test.csv" + manifest_content = b"text_fingerprint,split\nabc,test\n" + manifest_path.write_bytes(manifest_content) + output_path = tmp_path / "evaluation_records.json" + + monkeypatch.setattr( + evaluation, + "SPLIT_MANIFEST_PATH", + manifest_path, + ) + monkeypatch.setattr( + evaluation, + "EVALUATION_RECORDS_PATH", + output_path, + ) + + evaluation.save_evaluation_records( + [_record()], + dataset_fingerprint="dataset-fingerprint", + ) + + payload = json.loads(output_path.read_text(encoding="utf-8")) + + assert payload["split_manifest"] == manifest_path.name + assert payload["split_manifest_sha256"] == hashlib.sha256( + manifest_content + ).hexdigest() + assert payload["random_state"] == 42 + assert payload["positive_label"] == "phishing" + assert payload["source_split"] == "test" + assert payload["record_count"] == 1 + + +def test_requires_split_manifest_before_saving( + monkeypatch, + tmp_path: Path, +) -> None: + monkeypatch.setattr( + evaluation, + "SPLIT_MANIFEST_PATH", + tmp_path / "missing.csv", + ) + + try: + evaluation.save_evaluation_records( + [_record()], + dataset_fingerprint="dataset-fingerprint", + ) + except FileNotFoundError as exception: + assert "split manifest is required" in str(exception) + else: + raise AssertionError("missing split manifest must be rejected") From ab071674bcc5c2e127008d7f86bf08abdcff4187 Mon Sep 17 00:00:00 2001 From: pearseona Date: Sun, 16 Aug 2026 16:39:47 +0900 Subject: [PATCH 4/5] docs: clarify evaluatin denominators in reports --- .../SMSModel/hybrid_evaluation/reporting.py | 172 ++++++++++++++++-- .../hybrid_evaluation/test_reporting.py | 18 +- 2 files changed, 174 insertions(+), 16 deletions(-) diff --git a/data_science/SMSModel/hybrid_evaluation/reporting.py b/data_science/SMSModel/hybrid_evaluation/reporting.py index ddc8cf8..3264be0 100644 --- a/data_science/SMSModel/hybrid_evaluation/reporting.py +++ b/data_science/SMSModel/hybrid_evaluation/reporting.py @@ -206,7 +206,17 @@ def render_markdown_report(report: dict[str, Any]) -> str: f"- 생성 시각: `{report['generated_at']}`", f"- 평가 split: `{report['source_split']}`", f"- 샘플 수: `{report['sample_count']}`", + "- 라벨 분포: " + f"`normal={report['dataset']['normal_count']}`, " + f"`phishing={report['dataset']['phishing_count']}`", + f"- Positive label: `{report['dataset']['positive_label']}`", f"- 데이터셋 fingerprint: `{report['dataset_fingerprint']}`", + "- Split manifest: " + f"`{report['evaluation_provenance']['split_manifest']}`", + "- Split manifest SHA-256: " + f"`{report['evaluation_provenance']['split_manifest_sha256']}`", + "- Random state: " + f"`{report['evaluation_provenance']['random_state']}`", f"- LLM: `{report['model']['provider']}` / `{report['model']['model_id']}`", f"- Region: `{report['model']['region']}`", f"- 프롬프트 버전: `{report['model']['prompt_version']}`", @@ -214,6 +224,12 @@ def render_markdown_report(report: dict[str, Any]) -> str: f"`{report['stacking_artifact']['sha256']}`", "- Stacking artifact schema: " f"`{report['stacking_artifact']['schema_version']}`", + "- Stacking classification threshold: " + f"`{report['stacking_artifact']['classification_threshold']:.8f}`", + "- Threshold source split: " + f"`{report['stacking_artifact']['threshold_source_split']}`", + "- Threshold selection: " + f"`{report['stacking_artifact']['threshold_selection_metric']}`", "", "## 라우팅 정책", "", @@ -229,7 +245,10 @@ def render_markdown_report(report: dict[str, Any]) -> str: f"{report['sample_count']}` " f"(`{_percent(report['routing_policy']['test_uncertain_rate'])}`)", "", - "## 모드별 비교", + "## 전체 표본 기준 비교", + "", + "`UNKNOWN`을 포함한 전체 test 표본을 분모로 사용합니다. 사용 불가능한 " + "결과는 전체 정확도에서 실패로, 실제 피싱 표본에서는 미탐으로 계산합니다.", "", "| 지표 | Stacking only | Claude only | Hybrid |", "|---|---:|---:|---:|", @@ -253,6 +272,66 @@ def add_row(label: str, path: tuple[str, ...], formatter) -> None: f"| {label} | " + " | ".join(formatter(value) for value in values) + " |" ) + add_row( + "전체 표본 수", + ("classification", "full_dataset", "total_sample_count"), + str, + ) + add_row( + "결과 있음", + ("classification", "full_dataset", "available_count"), + str, + ) + add_row( + "결과 없음", + ("classification", "full_dataset", "unavailable_count"), + str, + ) + add_row("Availability", ("availability", "availability_rate"), _percent) + add_row( + "전체 정확도", + ("classification", "full_dataset", "accuracy"), + _format_ratio, + ) + add_row( + "피싱 탐지율", + ("classification", "full_dataset", "phishing_detection_rate"), + _format_ratio, + ) + add_row( + "정답", + ("classification", "full_dataset", "correct_count"), + str, + ) + add_row( + "오분류", + ("classification", "full_dataset", "incorrect_count"), + str, + ) + add_row( + "피싱 미탐", + ("classification", "full_dataset", "missed_phishing_count"), + str, + ) + + lines.extend( + [ + "", + "## 결과 성공 표본 기준 이진 분류", + "", + "아래 지표는 `UNKNOWN`을 제외한 결과이므로 Availability와 함께 " + "해석해야 합니다.", + "", + "| 지표 | Stacking only | Claude only | Hybrid |", + "|---|---:|---:|---:|", + ] + ) + + add_row( + "평가 표본 수", + ("classification", "available_only", "sample_count"), + lambda value: "N/A" if value is None else str(value), + ) for metric_name, label in ( ("accuracy", "Accuracy"), ("precision", "Precision"), @@ -266,6 +345,31 @@ def add_row(label: str, path: tuple[str, ...], formatter) -> None: _format_ratio, ) + for metric_name, label in ( + ("true_negative", "TN"), + ("false_positive", "FP"), + ("false_negative", "FN"), + ("true_positive", "TP"), + ): + add_row( + label, + ("classification", "available_only", metric_name), + lambda value: "N/A" if value is None else str(value), + ) + + lines.extend( + [ + "", + "## 운영·비용 비교", + "", + "Latency는 성공·실패·fallback을 모두 포함한 메시지 단위 " + "end-to-end 처리 시간입니다.", + "", + "| 지표 | Stacking only | Claude only | Hybrid |", + "|---|---:|---:|---:|", + ] + ) + add_row("평균 지연시간 (ms)", ("latency_ms", "average_ms"), _format_number) add_row("P50 지연시간 (ms)", ("latency_ms", "p50_ms"), _format_number) add_row("P95 지연시간 (ms)", ("latency_ms", "p95_ms"), _format_number) @@ -280,7 +384,6 @@ def add_row(label: str, path: tuple[str, ...], formatter) -> None: ("operations", "all_engines_unavailable_count"), str, ) - add_row("결과 없음", ("availability", "unavailable_count"), str) comparisons = report["comparisons"] lines.extend( @@ -325,8 +428,9 @@ def add_row(label: str, path: tuple[str, ...], formatter) -> None: "", "## 판정 정책", "", - "`UNKNOWN` 결과는 정상으로 간주하지 않습니다. 이진 분류 지표에서 제외하고 " - "각 모드의 `결과 없음` 건수로 별도 기록합니다.", + "`UNKNOWN` 결과는 정상으로 간주하지 않습니다. Available-only 이진 분류 " + "지표에서는 제외하지만 전체 표본 지표의 분모에는 유지하며, 실제 피싱의 " + "`UNKNOWN`은 미탐으로 계산합니다.", "", "## 재현 명령어", "", @@ -353,7 +457,7 @@ def add_row(label: str, path: tuple[str, ...], formatter) -> None: "data_science.SMSModel.run_hybrid_evaluation --collect", "```", "", - "## Issue #37 PR 3 체크리스트 (Bedrock/Claude)", + "## 평가 검증 체크리스트", "", "- [x] 고정된 test split에서 Stacking-only 평가", "- [x] AWS Bedrock Claude Haiku test 예측 수집", @@ -380,10 +484,20 @@ def render_csv_report(report: dict[str, Any]) -> str: output = io.StringIO(newline="") fieldnames = [ "generated_at", "source_split", "sample_count", "dataset_fingerprint", - "stacking_artifact_sha256", "llm_provider", "llm_model_id", "aws_region", - "prompt_version", "normal_probability_max", "phishing_probability_min", - "currency", "input_price_per_million", "output_price_per_million", "mode", - "accuracy", "precision", "recall", "f1", "f2", "average_latency_ms", + "normal_sample_count", "phishing_sample_count", "positive_label", + "split_manifest", "split_manifest_sha256", "random_state", + "stacking_artifact_sha256", "stacking_classification_threshold", + "threshold_source_split", "threshold_selection_metric", "llm_provider", + "llm_model_id", "aws_region", "prompt_version", "normal_probability_max", + "phishing_probability_min", "currency", "input_price_per_million", + "output_price_per_million", "mode", "available_sample_count", + "unavailable_sample_count", "availability_rate", "available_only_accuracy", + "available_only_precision", "available_only_recall", "available_only_f1", + "available_only_f2", "true_negative", "false_positive", "false_negative", + "true_positive", "full_dataset_accuracy", + "full_dataset_phishing_detection_rate", "full_dataset_correct_count", + "full_dataset_incorrect_count", "full_dataset_unavailable_count", + "average_latency_ms", "p50_latency_ms", "p95_latency_ms", "llm_call_rate", "input_tokens", "output_tokens", "cost_per_message", "fallback_count", "all_engines_unavailable_count", "unmeasured_token_usage_count", @@ -392,13 +506,29 @@ def render_csv_report(report: dict[str, Any]) -> str: writer.writeheader() for mode in _MODES: summary = report["modes"][mode.value] - classification = summary["classification"]["available_only"] or {} + available_only = summary["classification"]["available_only"] or {} + full_dataset = summary["classification"]["full_dataset"] writer.writerow({ "generated_at": report["generated_at"], "source_split": report["source_split"], "sample_count": report["sample_count"], "dataset_fingerprint": report["dataset_fingerprint"], + "normal_sample_count": report["dataset"]["normal_count"], + "phishing_sample_count": report["dataset"]["phishing_count"], + "positive_label": report["dataset"]["positive_label"], + "split_manifest": report["evaluation_provenance"]["split_manifest"], + "split_manifest_sha256": report["evaluation_provenance"]["split_manifest_sha256"], + "random_state": report["evaluation_provenance"]["random_state"], "stacking_artifact_sha256": report["stacking_artifact"]["sha256"], + "stacking_classification_threshold": report[ + "stacking_artifact" + ]["classification_threshold"], + "threshold_source_split": report[ + "stacking_artifact" + ]["threshold_source_split"], + "threshold_selection_metric": report[ + "stacking_artifact" + ]["threshold_selection_metric"], "llm_provider": report["model"]["provider"], "llm_model_id": report["model"]["model_id"], "aws_region": report["model"]["region"], @@ -409,11 +539,23 @@ def render_csv_report(report: dict[str, Any]) -> str: "input_price_per_million": report["pricing"]["input_price"], "output_price_per_million": report["pricing"]["output_price"], "mode": mode.value, - "accuracy": classification.get("accuracy"), - "precision": classification.get("precision"), - "recall": classification.get("recall"), - "f1": classification.get("f1"), - "f2": classification.get("f2"), + "available_sample_count": full_dataset["available_count"], + "unavailable_sample_count": full_dataset["unavailable_count"], + "availability_rate": summary["availability"]["availability_rate"], + "available_only_accuracy": available_only.get("accuracy"), + "available_only_precision": available_only.get("precision"), + "available_only_recall": available_only.get("recall"), + "available_only_f1": available_only.get("f1"), + "available_only_f2": available_only.get("f2"), + "true_negative": available_only.get("true_negative"), + "false_positive": available_only.get("false_positive"), + "false_negative": available_only.get("false_negative"), + "true_positive": available_only.get("true_positive"), + "full_dataset_accuracy": full_dataset["accuracy"], + "full_dataset_phishing_detection_rate": full_dataset["phishing_detection_rate"], + "full_dataset_correct_count": full_dataset["correct_count"], + "full_dataset_incorrect_count": full_dataset["incorrect_count"], + "full_dataset_unavailable_count": full_dataset["unavailable_count"], "average_latency_ms": summary["latency_ms"]["average_ms"], "p50_latency_ms": summary["latency_ms"]["p50_ms"], "p95_latency_ms": summary["latency_ms"]["p95_ms"], diff --git a/tests/data_science/SMSModel/hybrid_evaluation/test_reporting.py b/tests/data_science/SMSModel/hybrid_evaluation/test_reporting.py index 03197f1..1cb0acc 100644 --- a/tests/data_science/SMSModel/hybrid_evaluation/test_reporting.py +++ b/tests/data_science/SMSModel/hybrid_evaluation/test_reporting.py @@ -307,13 +307,29 @@ def test_renders_markdown_without_sample_identifiers() -> None: assert "Stacking artifact SHA-256" in markdown assert "목표 지표 충족 여부" in markdown assert "임계값 채택 결론" in markdown - assert "Issue #37 PR 3 체크리스트" in markdown + assert "전체 표본 기준 비교" in markdown + assert "결과 성공 표본 기준 이진 분류" in markdown + assert "Positive label: `phishing`" in markdown + assert "| TN |" in markdown + assert "| FP |" in markdown + assert "| FN |" in markdown + assert "| TP |" in markdown + assert "평가 검증 체크리스트" in markdown + assert "Issue #37 PR 3 체크리스트" not in markdown def test_renders_reproducible_summary_csv_without_message_data() -> None: csv_report = render_csv_report(_build()) assert "stacking_artifact_sha256" in csv_report + assert "available_only_accuracy" in csv_report + assert "full_dataset_accuracy" in csv_report + assert "full_dataset_phishing_detection_rate" in csv_report + assert "split_manifest_sha256" in csv_report + assert "true_negative" in csv_report + assert "false_positive" in csv_report + assert "false_negative" in csv_report + assert "true_positive" in csv_report assert "SELF_MODEL_ONLY" in csv_report assert "LLM_ONLY" in csv_report assert "HYBRID" in csv_report From 431011b4145dcb7b55b6334f8f4bc03b956a4c56 Mon Sep 17 00:00:00 2001 From: pearseona Date: Sun, 16 Aug 2026 16:43:56 +0900 Subject: [PATCH 5/5] test: regenerate unified evaluation reports --- .../hybrid_evaluation/comparison_report.csv | 8 +- .../hybrid_evaluation/comparison_report.json | 174 ++-- .../hybrid_evaluation/comparison_report.md | 63 +- .../hybrid_evaluation/evaluation_records.json | 762 +++++++++--------- 4 files changed, 557 insertions(+), 450 deletions(-) diff --git a/data_science/SMSModel/reports/hybrid_evaluation/comparison_report.csv b/data_science/SMSModel/reports/hybrid_evaluation/comparison_report.csv index c6ccd5f..d9b5369 100644 --- a/data_science/SMSModel/reports/hybrid_evaluation/comparison_report.csv +++ b/data_science/SMSModel/reports/hybrid_evaluation/comparison_report.csv @@ -1,4 +1,4 @@ -generated_at,source_split,sample_count,dataset_fingerprint,stacking_artifact_sha256,llm_provider,llm_model_id,aws_region,prompt_version,normal_probability_max,phishing_probability_min,currency,input_price_per_million,output_price_per_million,mode,accuracy,precision,recall,f1,f2,average_latency_ms,p50_latency_ms,p95_latency_ms,llm_call_rate,input_tokens,output_tokens,cost_per_message,fallback_count,all_engines_unavailable_count,unmeasured_token_usage_count -2026-08-13T13:15:48.374485+00:00,test,126,d10c1de722b2caa9f50e3d3873c9afaab7d9472fca304f1c5cd91390b0764d6f,fa99576f1d655625410ddb3b6b36ab55faca5a780f296dae9041e37593c063fe,AWS_BEDROCK,us.anthropic.claude-haiku-4-5-20251001-v1:0,us-east-1,smishing-v1:92102fecd96463d0d061cdf868df78f33de19ef86e44d508aae1d6c91897dee1,0.08716666259958943,0.59,USD,1.0,5.0,SELF_MODEL_ONLY,0.7857142857142857,0.7352941176470589,1.0,0.847457627118644,0.9328358208955224,122.72838413316224,31.06780000962317,45.36935011856258,0.0,0,0,0.0,0,0,0 -2026-08-13T13:15:48.374485+00:00,test,126,d10c1de722b2caa9f50e3d3873c9afaab7d9472fca304f1c5cd91390b0764d6f,fa99576f1d655625410ddb3b6b36ab55faca5a780f296dae9041e37593c063fe,AWS_BEDROCK,us.anthropic.claude-haiku-4-5-20251001-v1:0,us-east-1,smishing-v1:92102fecd96463d0d061cdf868df78f33de19ef86e44d508aae1d6c91897dee1,0.08716666259958943,0.59,USD,1.0,5.0,LLM_ONLY,0.8942307692307693,0.8571428571428571,1.0,0.9230769230769231,0.967741935483871,2409.793567458419,3075.032049953006,3771.050324946642,1.0,42508,27745,0.0014383571428571427,0,22,22 -2026-08-13T13:15:48.374485+00:00,test,126,d10c1de722b2caa9f50e3d3873c9afaab7d9472fca304f1c5cd91390b0764d6f,fa99576f1d655625410ddb3b6b36ab55faca5a780f296dae9041e37593c063fe,AWS_BEDROCK,us.anthropic.claude-haiku-4-5-20251001-v1:0,us-east-1,smishing-v1:92102fecd96463d0d061cdf868df78f33de19ef86e44d508aae1d6c91897dee1,0.08716666259958943,0.59,USD,1.0,5.0,HYBRID,0.873015873015873,0.8241758241758241,1.0,0.9036144578313253,0.959079283887468,466.0485682517793,32.63625002000481,3218.902824927587,0.23015873015873015,8342,4626,0.0002497777777777778,8,0,8 +generated_at,source_split,sample_count,dataset_fingerprint,normal_sample_count,phishing_sample_count,positive_label,split_manifest,split_manifest_sha256,random_state,stacking_artifact_sha256,stacking_classification_threshold,threshold_source_split,threshold_selection_metric,llm_provider,llm_model_id,aws_region,prompt_version,normal_probability_max,phishing_probability_min,currency,input_price_per_million,output_price_per_million,mode,available_sample_count,unavailable_sample_count,availability_rate,available_only_accuracy,available_only_precision,available_only_recall,available_only_f1,available_only_f2,true_negative,false_positive,false_negative,true_positive,full_dataset_accuracy,full_dataset_phishing_detection_rate,full_dataset_correct_count,full_dataset_incorrect_count,full_dataset_unavailable_count,average_latency_ms,p50_latency_ms,p95_latency_ms,llm_call_rate,input_tokens,output_tokens,cost_per_message,fallback_count,all_engines_unavailable_count,unmeasured_token_usage_count +2026-08-16T07:41:09.640776+00:00,test,126,d10c1de722b2caa9f50e3d3873c9afaab7d9472fca304f1c5cd91390b0764d6f,51,75,phishing,sms_split_v1.csv,4c33fcc749ca5c8ac4e6b35d19831e3ad8bffb02e72ac4d50d423d7ca19aa5f9,42,fa99576f1d655625410ddb3b6b36ab55faca5a780f296dae9041e37593c063fe,0.08844759684377157,validation,maximize_f2_subject_to_target_recall,AWS_BEDROCK,us.anthropic.claude-haiku-4-5-20251001-v1:0,us-east-1,smishing-v1:92102fecd96463d0d061cdf868df78f33de19ef86e44d508aae1d6c91897dee1,0.08716666259958943,0.59,USD,1.0,5.0,SELF_MODEL_ONLY,126,0,1.0,0.7857142857142857,0.7352941176470589,1.0,0.847457627118644,0.9328358208955224,24,27,0,75,0.7857142857142857,1.0,99,27,0,52.95694682244507,26.397299952805042,43.74754993477836,0.0,0,0,0.0,0,0,0 +2026-08-16T07:41:09.640776+00:00,test,126,d10c1de722b2caa9f50e3d3873c9afaab7d9472fca304f1c5cd91390b0764d6f,51,75,phishing,sms_split_v1.csv,4c33fcc749ca5c8ac4e6b35d19831e3ad8bffb02e72ac4d50d423d7ca19aa5f9,42,fa99576f1d655625410ddb3b6b36ab55faca5a780f296dae9041e37593c063fe,0.08844759684377157,validation,maximize_f2_subject_to_target_recall,AWS_BEDROCK,us.anthropic.claude-haiku-4-5-20251001-v1:0,us-east-1,smishing-v1:92102fecd96463d0d061cdf868df78f33de19ef86e44d508aae1d6c91897dee1,0.08716666259958943,0.59,USD,1.0,5.0,LLM_ONLY,104,22,0.8253968253968254,0.8942307692307693,0.8571428571428571,1.0,0.9230769230769231,0.967741935483871,27,11,0,66,0.7380952380952381,0.88,93,11,22,2409.7936579334505,3075.0416500261053,3771.0595999881625,1.0,42508,27745,0.0014383571428571427,0,22,22 +2026-08-16T07:41:09.640776+00:00,test,126,d10c1de722b2caa9f50e3d3873c9afaab7d9472fca304f1c5cd91390b0764d6f,51,75,phishing,sms_split_v1.csv,4c33fcc749ca5c8ac4e6b35d19831e3ad8bffb02e72ac4d50d423d7ca19aa5f9,42,fa99576f1d655625410ddb3b6b36ab55faca5a780f296dae9041e37593c063fe,0.08844759684377157,validation,maximize_f2_subject_to_target_recall,AWS_BEDROCK,us.anthropic.claude-haiku-4-5-20251001-v1:0,us-east-1,smishing-v1:92102fecd96463d0d061cdf868df78f33de19ef86e44d508aae1d6c91897dee1,0.08716666259958943,0.59,USD,1.0,5.0,HYBRID,126,0,1.0,0.873015873015873,0.8241758241758241,1.0,0.9036144578313253,0.959079283887468,35,16,0,75,0.873015873015873,1.0,110,16,0,462.66939683509844,28.347349958494306,3214.1587500381283,0.23015873015873015,8342,4626,0.0002497777777777778,8,0,8 diff --git a/data_science/SMSModel/reports/hybrid_evaluation/comparison_report.json b/data_science/SMSModel/reports/hybrid_evaluation/comparison_report.json index 2849ca5..c150fc1 100644 --- a/data_science/SMSModel/reports/hybrid_evaluation/comparison_report.json +++ b/data_science/SMSModel/reports/hybrid_evaluation/comparison_report.json @@ -1,19 +1,35 @@ { "classification_policy": { + "available_only_metrics": "Binary classification metrics use only records with an available normal or phishing prediction.", + "full_dataset_metrics": "Full-dataset accuracy uses every test sample; unavailable results count as unsuccessful outcomes. Phishing detection rate counts unavailable phishing samples as missed.", "positive_label": "phishing", - "unknown_handling": "UNKNOWN results are excluded from binary classification metrics and reported as unavailable_count." + "unknown_handling": "UNKNOWN results are excluded from available-only binary metrics, retained in full-dataset denominators, and reported as unavailable_count." }, "comparisons": { - "average_latency_reduction_rate": 0.8066022855462615, + "average_latency_reduction_rate": 0.8080045586841379, "cost_per_message_reduction_rate": 0.8263450916775642, + "hybrid_detection_rate_delta_vs_self_model": 0.0, "hybrid_f2_delta_vs_self_model": 0.026243462991945576, + "hybrid_full_accuracy_delta_vs_self_model": 0.08730158730158732, "hybrid_recall_delta_vs_self_model": 0.0, "llm_call_reduction_rate": 0.7698412698412699, - "p95_latency_reduction_rate": 0.14641743080606262 + "p95_latency_reduction_rate": 0.1476775519410466 + }, + "dataset": { + "normal_count": 51, + "phishing_count": 75, + "positive_label": "phishing", + "total_count": 126 }, "dataset_fingerprint": "d10c1de722b2caa9f50e3d3873c9afaab7d9472fca304f1c5cd91390b0764d6f", + "evaluation_provenance": { + "positive_label": "phishing", + "random_state": 42, + "split_manifest": "sms_split_v1.csv", + "split_manifest_sha256": "4c33fcc749ca5c8ac4e6b35d19831e3ad8bffb02e72ac4d50d423d7ca19aa5f9" + }, "evaluation_schema_version": 1, - "generated_at": "2026-08-13T13:15:48.374485+00:00", + "generated_at": "2026-08-16T07:41:09.640776+00:00", "model": { "model_id": "us.anthropic.claude-haiku-4-5-20251001-v1:0", "prompt_version": "smishing-v1:92102fecd96463d0d061cdf868df78f33de19ef86e44d508aae1d6c91897dee1", @@ -28,16 +44,31 @@ "unavailable_count": 0 }, "classification": { - "accuracy": 0.873015873015873, - "f1": 0.9036144578313253, - "f2": 0.959079283887468, - "false_negative": 0, - "false_positive": 16, - "precision": 0.8241758241758241, - "recall": 1.0, - "sample_count": 126, - "true_negative": 35, - "true_positive": 75 + "available_only": { + "accuracy": 0.873015873015873, + "f1": 0.9036144578313253, + "f2": 0.959079283887468, + "false_negative": 0, + "false_positive": 16, + "precision": 0.8241758241758241, + "recall": 1.0, + "sample_count": 126, + "true_negative": 35, + "true_positive": 75 + }, + "full_dataset": { + "accuracy": 0.873015873015873, + "actual_normal_count": 51, + "actual_phishing_count": 75, + "available_count": 126, + "correct_count": 110, + "detected_phishing_count": 75, + "incorrect_count": 16, + "missed_phishing_count": 0, + "phishing_detection_rate": 1.0, + "total_sample_count": 126, + "unavailable_count": 0 + } }, "cost": { "cost_per_message": 0.0002497777777777778, @@ -48,11 +79,11 @@ "unmeasured_call_count": 8 }, "latency_ms": { - "average_ms": 466.0485682517793, - "maximum_ms": 4036.308800043538, - "minimum_ms": 21.24790009111166, - "p50_ms": 32.63625002000481, - "p95_ms": 3218.902824927587, + "average_ms": 462.66939683509844, + "maximum_ms": 4035.335499966517, + "minimum_ms": 18.363599898293614, + "p50_ms": 28.347349958494306, + "p95_ms": 3214.1587500381283, "sample_count": 126 }, "operations": { @@ -72,16 +103,31 @@ "unavailable_count": 22 }, "classification": { - "accuracy": 0.8942307692307693, - "f1": 0.9230769230769231, - "f2": 0.967741935483871, - "false_negative": 0, - "false_positive": 11, - "precision": 0.8571428571428571, - "recall": 1.0, - "sample_count": 104, - "true_negative": 27, - "true_positive": 66 + "available_only": { + "accuracy": 0.8942307692307693, + "f1": 0.9230769230769231, + "f2": 0.967741935483871, + "false_negative": 0, + "false_positive": 11, + "precision": 0.8571428571428571, + "recall": 1.0, + "sample_count": 104, + "true_negative": 27, + "true_positive": 66 + }, + "full_dataset": { + "accuracy": 0.7380952380952381, + "actual_normal_count": 51, + "actual_phishing_count": 75, + "available_count": 104, + "correct_count": 93, + "detected_phishing_count": 66, + "incorrect_count": 11, + "missed_phishing_count": 9, + "phishing_detection_rate": 0.88, + "total_sample_count": 126, + "unavailable_count": 22 + } }, "cost": { "cost_per_message": 0.0014383571428571427, @@ -92,11 +138,11 @@ "unmeasured_call_count": 22 }, "latency_ms": { - "average_ms": 2409.793567458419, - "maximum_ms": 4065.016199890524, - "minimum_ms": 0.014800112694501877, - "p50_ms": 3075.032049953006, - "p95_ms": 3771.050324946642, + "average_ms": 2409.7936579334505, + "maximum_ms": 4065.0177999027073, + "minimum_ms": 0.011299969628453255, + "p50_ms": 3075.0416500261053, + "p95_ms": 3771.0595999881625, "sample_count": 126 }, "operations": { @@ -116,16 +162,31 @@ "unavailable_count": 0 }, "classification": { - "accuracy": 0.7857142857142857, - "f1": 0.847457627118644, - "f2": 0.9328358208955224, - "false_negative": 0, - "false_positive": 27, - "precision": 0.7352941176470589, - "recall": 1.0, - "sample_count": 126, - "true_negative": 24, - "true_positive": 75 + "available_only": { + "accuracy": 0.7857142857142857, + "f1": 0.847457627118644, + "f2": 0.9328358208955224, + "false_negative": 0, + "false_positive": 27, + "precision": 0.7352941176470589, + "recall": 1.0, + "sample_count": 126, + "true_negative": 24, + "true_positive": 75 + }, + "full_dataset": { + "accuracy": 0.7857142857142857, + "actual_normal_count": 51, + "actual_phishing_count": 75, + "available_count": 126, + "correct_count": 99, + "detected_phishing_count": 75, + "incorrect_count": 27, + "missed_phishing_count": 0, + "phishing_detection_rate": 1.0, + "total_sample_count": 126, + "unavailable_count": 0 + } }, "cost": { "cost_per_message": 0.0, @@ -136,11 +197,11 @@ "unmeasured_call_count": 0 }, "latency_ms": { - "average_ms": 122.72838413316224, - "maximum_ms": 11324.469800107181, - "minimum_ms": 17.560599837452173, - "p50_ms": 31.06780000962317, - "p95_ms": 45.36935011856258, + "average_ms": 52.95694682244507, + "maximum_ms": 3116.569299949333, + "minimum_ms": 10.648499941453338, + "p50_ms": 26.397299952805042, + "p95_ms": 43.74754993477836, "sample_count": 126 }, "operations": { @@ -161,7 +222,7 @@ "pricing_as_of": "2026-08-13", "unit": "per_1_million_tokens" }, - "report_schema_version": 2, + "report_schema_version": 3, "routing_policy": { "normal_probability_max": 0.08716666259958943, "phishing_probability_min": 0.59, @@ -173,17 +234,22 @@ "sample_count": 126, "source_split": "test", "stacking_artifact": { + "classification_threshold": 0.08844759684377157, "created_at": "2026-08-11T00:13:11.792602+00:00", "model_name": "stacking_phishing_classifier", + "random_state": 42, "schema_version": 1, - "sha256": "fa99576f1d655625410ddb3b6b36ab55faca5a780f296dae9041e37593c063fe" + "sha256": "fa99576f1d655625410ddb3b6b36ab55faca5a780f296dae9041e37593c063fe", + "target_recall": 0.95, + "threshold_selection_metric": "maximize_f2_subject_to_target_recall", + "threshold_source_split": "validation" }, "target_assessment": { "all_met": true, "items": [ { "actual": "1.0000", - "label": "Hybrid Recall", + "label": "Hybrid full-dataset phishing detection rate", "met": true, "target": ">= 0.9500" }, @@ -206,13 +272,13 @@ "target": "> 0%" }, { - "actual": "0.8066", + "actual": "0.8080", "label": "Average latency reduction", "met": true, "target": "> 0%" }, { - "actual": "0.1464", + "actual": "0.1477", "label": "P95 latency reduction", "met": true, "target": "> 0%" diff --git a/data_science/SMSModel/reports/hybrid_evaluation/comparison_report.md b/data_science/SMSModel/reports/hybrid_evaluation/comparison_report.md index d470414..e9d6fec 100644 --- a/data_science/SMSModel/reports/hybrid_evaluation/comparison_report.md +++ b/data_science/SMSModel/reports/hybrid_evaluation/comparison_report.md @@ -2,15 +2,23 @@ ## 평가 조건 -- 생성 시각: `2026-08-13T13:15:48.374485+00:00` +- 생성 시각: `2026-08-16T07:41:09.640776+00:00` - 평가 split: `test` - 샘플 수: `126` +- 라벨 분포: `normal=51`, `phishing=75` +- Positive label: `phishing` - 데이터셋 fingerprint: `d10c1de722b2caa9f50e3d3873c9afaab7d9472fca304f1c5cd91390b0764d6f` +- Split manifest: `sms_split_v1.csv` +- Split manifest SHA-256: `4c33fcc749ca5c8ac4e6b35d19831e3ad8bffb02e72ac4d50d423d7ca19aa5f9` +- Random state: `42` - LLM: `AWS_BEDROCK` / `us.anthropic.claude-haiku-4-5-20251001-v1:0` - Region: `us-east-1` - 프롬프트 버전: `smishing-v1:92102fecd96463d0d061cdf868df78f33de19ef86e44d508aae1d6c91897dee1` - Stacking artifact SHA-256: `fa99576f1d655625410ddb3b6b36ab55faca5a780f296dae9041e37593c063fe` - Stacking artifact schema: `1` +- Stacking classification threshold: `0.08844760` +- Threshold source split: `validation` +- Threshold selection: `maximize_f2_subject_to_target_recall` ## 라우팅 정책 @@ -20,18 +28,48 @@ - Validation 예상 LLM 호출률: `41.46%` - Test 실제 불확실 구간: `29/126` (`23.02%`) -## 모드별 비교 +## 전체 표본 기준 비교 + +`UNKNOWN`을 포함한 전체 test 표본을 분모로 사용합니다. 사용 불가능한 결과는 전체 정확도에서 실패로, 실제 피싱 표본에서는 미탐으로 계산합니다. + +| 지표 | Stacking only | Claude only | Hybrid | +|---|---:|---:|---:| +| 전체 표본 수 | 126 | 126 | 126 | +| 결과 있음 | 126 | 104 | 126 | +| 결과 없음 | 0 | 22 | 0 | +| Availability | 100.00% | 82.54% | 100.00% | +| 전체 정확도 | 0.7857 | 0.7381 | 0.8730 | +| 피싱 탐지율 | 1.0000 | 0.8800 | 1.0000 | +| 정답 | 99 | 93 | 110 | +| 오분류 | 27 | 11 | 16 | +| 피싱 미탐 | 0 | 9 | 0 | + +## 결과 성공 표본 기준 이진 분류 + +아래 지표는 `UNKNOWN`을 제외한 결과이므로 Availability와 함께 해석해야 합니다. | 지표 | Stacking only | Claude only | Hybrid | |---|---:|---:|---:| +| 평가 표본 수 | 126 | 104 | 126 | | Accuracy | 0.7857 | 0.8942 | 0.8730 | | Precision | 0.7353 | 0.8571 | 0.8242 | | Recall | 1.0000 | 1.0000 | 1.0000 | | F1 | 0.8475 | 0.9231 | 0.9036 | | F2 | 0.9328 | 0.9677 | 0.9591 | -| 평균 지연시간 (ms) | 122.73 | 2409.79 | 466.05 | -| P50 지연시간 (ms) | 31.07 | 3075.03 | 32.64 | -| P95 지연시간 (ms) | 45.37 | 3771.05 | 3218.90 | +| TN | 24 | 27 | 35 | +| FP | 27 | 11 | 16 | +| FN | 0 | 0 | 0 | +| TP | 75 | 66 | 75 | + +## 운영·비용 비교 + +Latency는 성공·실패·fallback을 모두 포함한 메시지 단위 end-to-end 처리 시간입니다. + +| 지표 | Stacking only | Claude only | Hybrid | +|---|---:|---:|---:| +| 평균 지연시간 (ms) | 52.96 | 2409.79 | 462.67 | +| P50 지연시간 (ms) | 26.40 | 3075.04 | 28.35 | +| P95 지연시간 (ms) | 43.75 | 3771.06 | 3214.16 | | LLM 호출률 | 0.00% | 100.00% | 23.02% | | 입력 token | 0 | 42508 | 8342 | | 출력 token | 0 | 27745 | 4626 | @@ -39,14 +77,13 @@ | 미측정 LLM 호출 | 0 | 22 | 8 | | Fallback | 0 | 0 | 8 | | 전체 엔진 실패 | 0 | 22 | 0 | -| 결과 없음 | 0 | 22 | 0 | ## Hybrid 개선 효과 - Claude-only 대비 LLM 호출 감소율: `76.98%` - Claude-only 대비 메시지당 비용 감소율: `82.63%` -- Claude-only 대비 평균 지연시간 감소율: `80.66%` -- Claude-only 대비 P95 지연시간 감소율: `14.64%` +- Claude-only 대비 평균 지연시간 감소율: `80.80%` +- Claude-only 대비 P95 지연시간 감소율: `14.77%` - Stacking-only 대비 Recall 변화: `+0.0000` - Stacking-only 대비 F2 변화: `+0.0262` @@ -54,12 +91,12 @@ | 목표 | 기준 | 실제 | 결과 | |---|---:|---:|:---:| -| Hybrid Recall | >= 0.9500 | 1.0000 | 충족 | +| Hybrid full-dataset phishing detection rate | >= 0.9500 | 1.0000 | 충족 | | Hybrid F2 vs Stacking-only | >= 0 delta | 0.0262 | 충족 | | Claude call reduction | > 0% | 0.7698 | 충족 | | Cost reduction | > 0% | 0.8263 | 충족 | -| Average latency reduction | > 0% | 0.8066 | 충족 | -| P95 latency reduction | > 0% | 0.1464 | 충족 | +| Average latency reduction | > 0% | 0.8080 | 충족 | +| P95 latency reduction | > 0% | 0.1477 | 충족 | - 충족: `6/6` - P95 지연시간 감소 목표는 미충족이며 결과를 그대로 기록했습니다. @@ -78,7 +115,7 @@ ## 판정 정책 -`UNKNOWN` 결과는 정상으로 간주하지 않습니다. 이진 분류 지표에서 제외하고 각 모드의 `결과 없음` 건수로 별도 기록합니다. +`UNKNOWN` 결과는 정상으로 간주하지 않습니다. Available-only 이진 분류 지표에서는 제외하지만 전체 표본 지표의 분모에는 유지하며, 실제 피싱의 `UNKNOWN`은 미탐으로 계산합니다. ## 재현 명령어 @@ -100,7 +137,7 @@ $env:AWS_PROFILE = "safefam-dev" & .\.venv\Scripts\python.exe -m data_science.SMSModel.run_hybrid_evaluation --collect ``` -## Issue #37 PR 3 체크리스트 (Bedrock/Claude) +## 평가 검증 체크리스트 - [x] 고정된 test split에서 Stacking-only 평가 - [x] AWS Bedrock Claude Haiku test 예측 수집 diff --git a/data_science/SMSModel/reports/hybrid_evaluation/evaluation_records.json b/data_science/SMSModel/reports/hybrid_evaluation/evaluation_records.json index dea6de5..f78c556 100644 --- a/data_science/SMSModel/reports/hybrid_evaluation/evaluation_records.json +++ b/data_science/SMSModel/reports/hybrid_evaluation/evaluation_records.json @@ -2,7 +2,9 @@ "dataset_fingerprint": "d10c1de722b2caa9f50e3d3873c9afaab7d9472fca304f1c5cd91390b0764d6f", "evaluation_schema_version": 1, "model_id": "us.anthropic.claude-haiku-4-5-20251001-v1:0", + "positive_label": "phishing", "prompt_version": "smishing-v1:92102fecd96463d0d061cdf868df78f33de19ef86e44d508aae1d6c91897dee1", + "random_state": 42, "record_count": 378, "records": [ { @@ -12,7 +14,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 11324.469800107181, + "latency_ms": 3116.569299949333, "llm_available": false, "llm_called": false, "llm_model": null, @@ -32,7 +34,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 696, - "latency_ms": 3778.0577999744564, + "latency_ms": 3778.069299945608, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -52,7 +54,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 53.65020013414323, + "latency_ms": 60.05800003185868, "llm_available": false, "llm_called": false, "llm_model": null, @@ -72,7 +74,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 27.626499999314547, + "latency_ms": 31.07809997163713, "llm_available": false, "llm_called": false, "llm_model": null, @@ -92,7 +94,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 366, - "latency_ms": 3699.0348000321537, + "latency_ms": 3699.036000041291, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -112,7 +114,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 32.70959993824363, + "latency_ms": 27.32670004479587, "llm_available": false, "llm_called": false, "llm_model": null, @@ -132,7 +134,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 31.25589992851019, + "latency_ms": 32.6554998755455, "llm_available": false, "llm_called": false, "llm_model": null, @@ -152,7 +154,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 353, - "latency_ms": 2903.0369001645595, + "latency_ms": 2903.025700079277, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -172,7 +174,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 34.827199997380376, + "latency_ms": 32.58150001056492, "llm_available": false, "llm_called": false, "llm_model": null, @@ -192,7 +194,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 35.5555999558419, + "latency_ms": 37.03580005094409, "llm_available": false, "llm_called": false, "llm_model": null, @@ -212,7 +214,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 373, - "latency_ms": 3496.0271000899374, + "latency_ms": 3496.0193000305444, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -232,7 +234,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 37.17979998327792, + "latency_ms": 35.35589994862676, "llm_available": false, "llm_called": false, "llm_model": null, @@ -252,7 +254,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 44.63090002536774, + "latency_ms": 46.11480003222823, "llm_available": false, "llm_called": false, "llm_model": null, @@ -272,7 +274,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 514, - "latency_ms": 3218.027999980375, + "latency_ms": 3218.1085000112653, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -292,7 +294,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 514, - "latency_ms": 3256.982499856502, + "latency_ms": 3244.451300131157, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -312,7 +314,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 34.96149997226894, + "latency_ms": 32.70459990017116, "llm_available": false, "llm_called": false, "llm_model": null, @@ -332,7 +334,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 366, - "latency_ms": 3179.0372999347746, + "latency_ms": 3179.038299942389, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -352,7 +354,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 34.99999991618097, + "latency_ms": 28.798799961805344, "llm_available": false, "llm_called": false, "llm_model": null, @@ -372,7 +374,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 44.213199988007545, + "latency_ms": 43.3288998901844, "llm_available": false, "llm_called": false, "llm_model": null, @@ -392,7 +394,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 496, - "latency_ms": 3079.024300068617, + "latency_ms": 3079.030299881473, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -412,7 +414,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 40.61819985508919, + "latency_ms": 36.2009999807924, "llm_available": false, "llm_called": false, "llm_model": null, @@ -432,7 +434,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 40.162499994039536, + "latency_ms": 41.798199992626905, "llm_available": false, "llm_called": false, "llm_model": null, @@ -452,7 +454,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 493, - "latency_ms": 3688.0348000321537, + "latency_ms": 3688.0394999515265, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -472,7 +474,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 37.43450017645955, + "latency_ms": 38.95479999482632, "llm_available": false, "llm_called": false, "llm_model": null, @@ -492,7 +494,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 47.75230004452169, + "latency_ms": 51.17890005931258, "llm_available": false, "llm_called": false, "llm_model": null, @@ -512,7 +514,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 682, - "latency_ms": 3989.034200027585, + "latency_ms": 3989.029999995604, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -532,7 +534,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 682, - "latency_ms": 4036.308800043538, + "latency_ms": 4035.335499966517, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -552,7 +554,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 33.618499990552664, + "latency_ms": 28.45460013486445, "llm_available": false, "llm_called": false, "llm_model": null, @@ -572,7 +574,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 413, - "latency_ms": 3161.03039999865, + "latency_ms": 3161.0275999773294, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -592,7 +594,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 413, - "latency_ms": 3199.700200151652, + "latency_ms": 3187.4445000793785, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -612,7 +614,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 42.77529986575246, + "latency_ms": 31.793199945241213, "llm_available": false, "llm_called": false, "llm_model": null, @@ -632,7 +634,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 543, - "latency_ms": 2112.026699854061, + "latency_ms": 2112.0297999940813, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -652,7 +654,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 47.10779991000891, + "latency_ms": 33.40389998629689, "llm_available": false, "llm_called": false, "llm_model": null, @@ -672,7 +674,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 33.3256998565048, + "latency_ms": 25.043000001460314, "llm_available": false, "llm_called": false, "llm_model": null, @@ -692,7 +694,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 372, - "latency_ms": 3119.123200006783, + "latency_ms": 3119.030600000173, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -712,7 +714,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 29.896499821916223, + "latency_ms": 26.2900001835078, "llm_available": false, "llm_called": false, "llm_model": null, @@ -732,7 +734,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 44.881199952214956, + "latency_ms": 39.75740005262196, "llm_available": false, "llm_called": false, "llm_model": null, @@ -752,7 +754,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 697, - "latency_ms": 3527.0378999393433, + "latency_ms": 3527.0310000032187, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -772,7 +774,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 49.994600005447865, + "latency_ms": 39.885600097477436, "llm_available": false, "llm_called": false, "llm_model": null, @@ -792,7 +794,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 30.313500203192234, + "latency_ms": 26.18589997291565, "llm_available": false, "llm_called": false, "llm_model": null, @@ -812,7 +814,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 390, - "latency_ms": 1936.0348999164999, + "latency_ms": 1936.0310998875648, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -832,7 +834,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 390, - "latency_ms": 1963.349199866876, + "latency_ms": 1968.2859999723732, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -852,7 +854,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 23.894699988886714, + "latency_ms": 27.301999973133206, "llm_available": false, "llm_called": false, "llm_model": null, @@ -872,7 +874,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 359, - "latency_ms": 3077.0358999241143, + "latency_ms": 3077.042000086978, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -892,7 +894,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 30.343399848788977, + "latency_ms": 32.47640002518892, "llm_available": false, "llm_called": false, "llm_model": null, @@ -912,7 +914,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 26.465100003406405, + "latency_ms": 27.213500114157796, "llm_available": false, "llm_called": false, "llm_model": null, @@ -932,7 +934,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 362, - "latency_ms": 3394.016699777916, + "latency_ms": 3394.0295999925584, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -952,7 +954,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 362, - "latency_ms": 3421.6593999005854, + "latency_ms": 3417.9419999998063, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -972,7 +974,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 27.639799984171987, + "latency_ms": 24.589499924331903, "llm_available": false, "llm_called": false, "llm_model": null, @@ -992,7 +994,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 367, - "latency_ms": 3980.0167000107467, + "latency_ms": 3980.0295001082122, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -1012,7 +1014,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 26.92830003798008, + "latency_ms": 24.915800197049975, "llm_available": false, "llm_called": false, "llm_model": null, @@ -1032,7 +1034,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 28.33539992570877, + "latency_ms": 24.70399998128414, "llm_available": false, "llm_called": false, "llm_model": null, @@ -1052,7 +1054,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 351, - "latency_ms": 3109.0179999042302, + "latency_ms": 3109.0291999895126, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -1072,7 +1074,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 29.89340014755726, + "latency_ms": 23.955400101840496, "llm_available": false, "llm_called": false, "llm_model": null, @@ -1092,7 +1094,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 40.28040007688105, + "latency_ms": 32.11469994857907, "llm_available": false, "llm_called": false, "llm_model": null, @@ -1112,7 +1114,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 516, - "latency_ms": 3191.01830002293, + "latency_ms": 3191.0374000519514, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -1132,7 +1134,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 516, - "latency_ms": 3225.3036998528987, + "latency_ms": 3223.0635000243783, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -1152,7 +1154,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 27.729899855330586, + "latency_ms": 25.712300091981888, "llm_available": false, "llm_called": false, "llm_model": null, @@ -1172,7 +1174,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 374, - "latency_ms": 3432.036400044337, + "latency_ms": 3432.040899962187, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -1192,7 +1194,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 33.08860003016889, + "latency_ms": 26.195500046014786, "llm_available": false, "llm_called": false, "llm_model": null, @@ -1212,7 +1214,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 38.54720015078783, + "latency_ms": 33.81590009666979, "llm_available": false, "llm_called": false, "llm_model": null, @@ -1232,7 +1234,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 497, - "latency_ms": 3610.0336000230163, + "latency_ms": 3610.0308000016958, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -1252,7 +1254,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 40.098399855196476, + "latency_ms": 33.307700185105205, "llm_available": false, "llm_called": false, "llm_model": null, @@ -1272,7 +1274,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 30.227600131183863, + "latency_ms": 27.255999855697155, "llm_available": false, "llm_called": false, "llm_model": null, @@ -1292,7 +1294,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 344, - "latency_ms": 1583.0271000899374, + "latency_ms": 1583.036200042814, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -1312,7 +1314,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 34.97970011085272, + "latency_ms": 32.72000001743436, "llm_available": false, "llm_called": false, "llm_model": null, @@ -1332,7 +1334,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 29.12249998189509, + "latency_ms": 26.188000105321407, "llm_available": false, "llm_called": false, "llm_model": null, @@ -1352,7 +1354,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 364, - "latency_ms": 1814.02419995144, + "latency_ms": 1814.0475000124425, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -1372,7 +1374,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 364, - "latency_ms": 1843.6048000454903, + "latency_ms": 1844.574900098145, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -1392,7 +1394,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 29.200500110164285, + "latency_ms": 30.582200037315488, "llm_available": false, "llm_called": false, "llm_model": null, @@ -1412,7 +1414,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 354, - "latency_ms": 2789.0269000884145, + "latency_ms": 2789.0310000032187, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -1432,7 +1434,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 33.95990002900362, + "latency_ms": 28.176800115033984, "llm_available": false, "llm_called": false, "llm_model": null, @@ -1452,7 +1454,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 53.51470015011728, + "latency_ms": 22.53079996444285, "llm_available": false, "llm_called": false, "llm_model": null, @@ -1472,7 +1474,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 347, - "latency_ms": 1696.0175000168383, + "latency_ms": 1696.02869986929, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -1492,7 +1494,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 36.27799986861646, + "latency_ms": 25.175699964165688, "llm_available": false, "llm_called": false, "llm_model": null, @@ -1512,7 +1514,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 44.25129992887378, + "latency_ms": 35.01029987819493, "llm_available": false, "llm_called": false, "llm_model": null, @@ -1532,7 +1534,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 497, - "latency_ms": 3255.0654999166727, + "latency_ms": 3255.032499898225, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -1552,7 +1554,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 34.67939980328083, + "latency_ms": 39.81200000271201, "llm_available": false, "llm_called": false, "llm_model": null, @@ -1572,7 +1574,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 31.97259991429746, + "latency_ms": 36.46789980120957, "llm_available": false, "llm_called": false, "llm_model": null, @@ -1592,7 +1594,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 497, - "latency_ms": 3483.0383998267353, + "latency_ms": 3483.0353001523763, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -1612,7 +1614,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 36.09750000759959, + "latency_ms": 33.86530000716448, "llm_available": false, "llm_called": false, "llm_model": null, @@ -1632,7 +1634,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 20.570799941197038, + "latency_ms": 26.449699886143208, "llm_available": false, "llm_called": false, "llm_model": null, @@ -1652,7 +1654,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 353, - "latency_ms": 2241.024799956009, + "latency_ms": 2241.0372999347746, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -1672,7 +1674,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 28.963500168174505, + "latency_ms": 27.733599999919534, "llm_available": false, "llm_called": false, "llm_model": null, @@ -1692,7 +1694,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 29.538500122725964, + "latency_ms": 27.321900008246303, "llm_available": false, "llm_called": false, "llm_model": null, @@ -1712,7 +1714,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 401, - "latency_ms": 2330.1018999610096, + "latency_ms": 2330.0314000062644, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -1732,7 +1734,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 22.92190003208816, + "latency_ms": 36.817300133407116, "llm_available": false, "llm_called": false, "llm_model": null, @@ -1752,7 +1754,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 17.560599837452173, + "latency_ms": 32.93260000646114, "llm_available": false, "llm_called": false, "llm_model": null, @@ -1772,7 +1774,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 374, - "latency_ms": 3354.01970003359, + "latency_ms": 3354.029900111258, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -1792,7 +1794,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 27.415999909862876, + "latency_ms": 25.715600000694394, "llm_available": false, "llm_called": false, "llm_model": null, @@ -1812,7 +1814,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 27.571599930524826, + "latency_ms": 25.65069985575974, "llm_available": false, "llm_called": false, "llm_model": null, @@ -1832,7 +1834,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 376, - "latency_ms": 3232.0287001021206, + "latency_ms": 3232.028899870813, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -1852,7 +1854,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 376, - "latency_ms": 3258.1681999545544, + "latency_ms": 3258.968899881467, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -1872,7 +1874,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 31.22920007444918, + "latency_ms": 25.84209991618991, "llm_available": false, "llm_called": false, "llm_model": null, @@ -1892,7 +1894,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 374, - "latency_ms": 1568.052200164646, + "latency_ms": 1568.028899870813, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -1912,7 +1914,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 30.45299998484552, + "latency_ms": 28.407699894160032, "llm_available": false, "llm_called": false, "llm_model": null, @@ -1932,7 +1934,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 39.457499980926514, + "latency_ms": 33.70780008845031, "llm_available": false, "llm_called": false, "llm_model": null, @@ -1952,7 +1954,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 0.024399952962994576, + "latency_ms": 0.025900080800056458, "llm_available": false, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -1972,7 +1974,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 35.77980003319681, + "latency_ms": 36.722500110045075, "llm_available": false, "llm_called": false, "llm_model": null, @@ -1992,7 +1994,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 27.131799841299653, + "latency_ms": 21.816899999976158, "llm_available": false, "llm_called": false, "llm_model": null, @@ -2012,7 +2014,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 347, - "latency_ms": 2142.026199966669, + "latency_ms": 2142.040600076318, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -2032,7 +2034,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 347, - "latency_ms": 2187.5684999469668, + "latency_ms": 2169.1439000498503, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -2052,7 +2054,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 40.80309998244047, + "latency_ms": 33.98469998501241, "llm_available": false, "llm_called": false, "llm_model": null, @@ -2072,7 +2074,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 0.02019992098212242, + "latency_ms": 0.024999957531690598, "llm_available": false, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -2092,7 +2094,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 40.066099958494306, + "latency_ms": 34.88050005398691, "llm_available": false, "llm_called": false, "llm_model": null, @@ -2112,7 +2114,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 26.336600072681904, + "latency_ms": 26.58480009995401, "llm_available": false, "llm_called": false, "llm_model": null, @@ -2132,7 +2134,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 332, - "latency_ms": 1662.0307001173496, + "latency_ms": 1662.040400074795, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -2152,7 +2154,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 332, - "latency_ms": 1689.765000006184, + "latency_ms": 1685.7167999148369, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -2172,7 +2174,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 39.63320003822446, + "latency_ms": 29.794800095260143, "llm_available": false, "llm_called": false, "llm_model": null, @@ -2192,7 +2194,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 0.0320998951792717, + "latency_ms": 0.04449998959898949, "llm_available": false, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -2212,7 +2214,7 @@ "expected_label": "normal", "fallback_applied": true, "input_tokens": null, - "latency_ms": 29.376700054854155, + "latency_ms": 31.11350000835955, "llm_available": false, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -2232,7 +2234,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 42.87070012651384, + "latency_ms": 48.80630015395582, "llm_available": false, "llm_called": false, "llm_model": null, @@ -2252,7 +2254,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 610, - "latency_ms": 3537.0259000808, + "latency_ms": 3537.0414999667555, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -2272,7 +2274,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 43.84759999811649, + "latency_ms": 49.788299947977066, "llm_available": false, "llm_called": false, "llm_model": null, @@ -2292,7 +2294,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 30.931900022551417, + "latency_ms": 35.61750007793307, "llm_available": false, "llm_called": false, "llm_model": null, @@ -2312,7 +2314,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 362, - "latency_ms": 3109.028299866244, + "latency_ms": 3109.032499898225, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -2332,7 +2334,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 31.99400007724762, + "latency_ms": 38.52619999088347, "llm_available": false, "llm_called": false, "llm_model": null, @@ -2352,7 +2354,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 26.476799976080656, + "latency_ms": 32.30770002119243, "llm_available": false, "llm_called": false, "llm_model": null, @@ -2372,7 +2374,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 0.02319994382560253, + "latency_ms": 0.034499913454055786, "llm_available": false, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -2392,7 +2394,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 30.070300213992596, + "latency_ms": 34.68780010007322, "llm_available": false, "llm_called": false, "llm_model": null, @@ -2412,7 +2414,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 44.7993001434952, + "latency_ms": 50.36290013231337, "llm_available": false, "llm_called": false, "llm_model": null, @@ -2432,7 +2434,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 496, - "latency_ms": 3427.0285001005977, + "latency_ms": 3427.040200073272, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -2452,7 +2454,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 39.202600019052625, + "latency_ms": 37.95290016569197, "llm_available": false, "llm_called": false, "llm_model": null, @@ -2472,7 +2474,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 45.142099959775805, + "latency_ms": 47.31189995072782, "llm_available": false, "llm_called": false, "llm_model": null, @@ -2492,7 +2494,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 778, - "latency_ms": 3164.026699854061, + "latency_ms": 3164.0414999667555, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -2512,7 +2514,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 53.63360000774264, + "latency_ms": 46.277600107714534, "llm_available": false, "llm_called": false, "llm_model": null, @@ -2532,7 +2534,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 29.3936999514699, + "latency_ms": 27.65880012884736, "llm_available": false, "llm_called": false, "llm_model": null, @@ -2552,7 +2554,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 0.02369983121752739, + "latency_ms": 0.023799948394298553, "llm_available": false, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -2572,7 +2574,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 29.34310003183782, + "latency_ms": 25.992900133132935, "llm_available": false, "llm_called": false, "llm_model": null, @@ -2592,7 +2594,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 33.60740002244711, + "latency_ms": 27.303499868139625, "llm_available": false, "llm_called": false, "llm_model": null, @@ -2612,7 +2614,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 356, - "latency_ms": 1920.0164001248777, + "latency_ms": 1920.0390000641346, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -2632,7 +2634,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 356, - "latency_ms": 1953.6091998033226, + "latency_ms": 1950.4095998872072, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -2652,7 +2654,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 49.27409999072552, + "latency_ms": 38.85869984515011, "llm_available": false, "llm_called": false, "llm_model": null, @@ -2672,7 +2674,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 0.016399892047047615, + "latency_ms": 0.0342000275850296, "llm_available": false, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -2692,7 +2694,7 @@ "expected_label": "normal", "fallback_applied": true, "input_tokens": null, - "latency_ms": 43.418999994173646, + "latency_ms": 42.588300071656704, "llm_available": false, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -2712,7 +2714,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 39.34410004876554, + "latency_ms": 41.84840014204383, "llm_available": false, "llm_called": false, "llm_model": null, @@ -2732,7 +2734,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 497, - "latency_ms": 3593.0289999879897, + "latency_ms": 3593.0376000534743, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -2752,7 +2754,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 42.59999981150031, + "latency_ms": 37.58679982274771, "llm_available": false, "llm_called": false, "llm_model": null, @@ -2772,7 +2774,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 40.199599927291274, + "latency_ms": 36.087899934500456, "llm_available": false, "llm_called": false, "llm_model": null, @@ -2792,7 +2794,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 496, - "latency_ms": 3508.03629992716, + "latency_ms": 3508.036200042814, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -2812,7 +2814,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 44.42599997855723, + "latency_ms": 37.45770012028515, "llm_available": false, "llm_called": false, "llm_model": null, @@ -2832,7 +2834,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 32.979700015857816, + "latency_ms": 27.526099933311343, "llm_available": false, "llm_called": false, "llm_model": null, @@ -2852,7 +2854,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 381, - "latency_ms": 1398.040000071749, + "latency_ms": 1398.0293001066893, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -2872,7 +2874,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 29.45709996856749, + "latency_ms": 27.798600029200315, "llm_available": false, "llm_called": false, "llm_model": null, @@ -2892,7 +2894,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 28.077400056645274, + "latency_ms": 27.48079993762076, "llm_available": false, "llm_called": false, "llm_model": null, @@ -2912,7 +2914,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 375, - "latency_ms": 3867.013899989426, + "latency_ms": 3867.0293001066893, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -2932,7 +2934,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 30.292500043287873, + "latency_ms": 26.80359990336001, "llm_available": false, "llm_called": false, "llm_model": null, @@ -2952,7 +2954,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 25.72060003876686, + "latency_ms": 24.775200057774782, "llm_available": false, "llm_called": false, "llm_model": null, @@ -2972,7 +2974,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 353, - "latency_ms": 1452.0375998206437, + "latency_ms": 1452.0314000062644, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -2992,7 +2994,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 31.44000004976988, + "latency_ms": 26.91599982790649, "llm_available": false, "llm_called": false, "llm_model": null, @@ -3012,7 +3014,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 31.892100116237998, + "latency_ms": 27.311400044709444, "llm_available": false, "llm_called": false, "llm_model": null, @@ -3032,7 +3034,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 0.024999957531690598, + "latency_ms": 0.03320001997053623, "llm_available": false, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -3052,7 +3054,7 @@ "expected_label": "normal", "fallback_applied": true, "input_tokens": null, - "latency_ms": 33.30060001462698, + "latency_ms": 27.367099886760116, "llm_available": false, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -3072,7 +3074,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 35.010999999940395, + "latency_ms": 31.016299966722727, "llm_available": false, "llm_called": false, "llm_model": null, @@ -3092,7 +3094,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 354, - "latency_ms": 3073.028199981898, + "latency_ms": 3073.0412999652326, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -3112,7 +3114,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 33.552899956703186, + "latency_ms": 28.88209978118539, "llm_available": false, "llm_called": false, "llm_model": null, @@ -3132,7 +3134,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 37.861399818211794, + "latency_ms": 33.697899896651506, "llm_available": false, "llm_called": false, "llm_model": null, @@ -3152,7 +3154,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 0.030800001695752144, + "latency_ms": 0.02450007013976574, "llm_available": false, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -3172,7 +3174,7 @@ "expected_label": "normal", "fallback_applied": true, "input_tokens": null, - "latency_ms": 37.53950004465878, + "latency_ms": 33.42360001988709, "llm_available": false, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -3192,7 +3194,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 31.64180018939078, + "latency_ms": 34.51679996214807, "llm_available": false, "llm_called": false, "llm_model": null, @@ -3212,7 +3214,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 360, - "latency_ms": 1518.0231999438256, + "latency_ms": 1518.029900111258, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -3232,7 +3234,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 29.25450005568564, + "latency_ms": 24.995099985972047, "llm_available": false, "llm_called": false, "llm_model": null, @@ -3252,7 +3254,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 32.977999886497855, + "latency_ms": 32.41600003093481, "llm_available": false, "llm_called": false, "llm_model": null, @@ -3272,7 +3274,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 497, - "latency_ms": 3550.028300099075, + "latency_ms": 3550.0291999895126, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -3292,7 +3294,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 38.16820005886257, + "latency_ms": 33.043700037524104, "llm_available": false, "llm_called": false, "llm_model": null, @@ -3312,7 +3314,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 32.8727001324296, + "latency_ms": 24.455799954012036, "llm_available": false, "llm_called": false, "llm_model": null, @@ -3332,7 +3334,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 351, - "latency_ms": 2907.0318000093102, + "latency_ms": 2907.0312000047415, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -3352,7 +3354,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 28.95070007070899, + "latency_ms": 28.28700002282858, "llm_available": false, "llm_called": false, "llm_model": null, @@ -3372,7 +3374,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 40.66170006990433, + "latency_ms": 29.27369996905327, "llm_available": false, "llm_called": false, "llm_model": null, @@ -3392,7 +3394,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 463, - "latency_ms": 3875.034400029108, + "latency_ms": 3875.030299881473, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -3412,7 +3414,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 463, - "latency_ms": 3909.15589989163, + "latency_ms": 3910.5869999621063, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -3432,7 +3434,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 42.87770017981529, + "latency_ms": 43.887099949643016, "llm_available": false, "llm_called": false, "llm_model": null, @@ -3452,7 +3454,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 496, - "latency_ms": 3645.035800039768, + "latency_ms": 3645.031500123441, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -3472,7 +3474,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 33.96090003661811, + "latency_ms": 34.14240013808012, "llm_available": false, "llm_called": false, "llm_model": null, @@ -3492,7 +3494,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 36.55870002694428, + "latency_ms": 29.132300056517124, "llm_available": false, "llm_called": false, "llm_model": null, @@ -3512,7 +3514,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 359, - "latency_ms": 3143.0173000153154, + "latency_ms": 3143.0310000032187, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -3532,7 +3534,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 30.144199961796403, + "latency_ms": 29.606400057673454, "llm_available": false, "llm_called": false, "llm_model": null, @@ -3552,7 +3554,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 23.841100046411157, + "latency_ms": 24.36690009199083, "llm_available": false, "llm_called": false, "llm_model": null, @@ -3572,7 +3574,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 338, - "latency_ms": 1618.0165000092238, + "latency_ms": 1618.02869986929, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -3592,7 +3594,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 338, - "latency_ms": 1650.334699993953, + "latency_ms": 1642.9862000346184, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -3612,7 +3614,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 46.78460001014173, + "latency_ms": 34.84140010550618, "llm_available": false, "llm_called": false, "llm_model": null, @@ -3632,7 +3634,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 0.024700071662664413, + "latency_ms": 0.011299969628453255, "llm_available": false, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -3652,7 +3654,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 38.07179979048669, + "latency_ms": 31.73330007120967, "llm_available": false, "llm_called": false, "llm_model": null, @@ -3672,7 +3674,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 28.097399976104498, + "latency_ms": 23.93839997239411, "llm_available": false, "llm_called": false, "llm_model": null, @@ -3692,7 +3694,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 349, - "latency_ms": 2058.024100067094, + "latency_ms": 2058.0309001188725, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -3712,7 +3714,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 349, - "latency_ms": 2085.37919986248, + "latency_ms": 2081.7150001339614, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -3732,7 +3734,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 37.158000050112605, + "latency_ms": 30.21460003219545, "llm_available": false, "llm_called": false, "llm_model": null, @@ -3752,7 +3754,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 497, - "latency_ms": 3336.0275999773294, + "latency_ms": 3336.0285001005977, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -3772,7 +3774,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 35.18030000850558, + "latency_ms": 33.464600099250674, "llm_available": false, "llm_called": false, "llm_model": null, @@ -3792,7 +3794,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 25.068699847906828, + "latency_ms": 24.334700079634786, "llm_available": false, "llm_called": false, "llm_model": null, @@ -3812,7 +3814,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 352, - "latency_ms": 1887.033700140193, + "latency_ms": 1887.0306000001729, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -3832,7 +3834,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 352, - "latency_ms": 1912.4579000175, + "latency_ms": 1907.9276999812573, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -3852,7 +3854,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 25.610400130972266, + "latency_ms": 16.260999953374267, "llm_available": false, "llm_called": false, "llm_model": null, @@ -3872,7 +3874,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 354, - "latency_ms": 2772.028300099075, + "latency_ms": 2772.0350999180228, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -3892,7 +3894,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 24.933900218456984, + "latency_ms": 27.310999808833003, "llm_available": false, "llm_called": false, "llm_model": null, @@ -3912,7 +3914,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 34.55250011757016, + "latency_ms": 34.388099797070026, "llm_available": false, "llm_called": false, "llm_model": null, @@ -3932,7 +3934,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 0.030199997127056122, + "latency_ms": 0.0241999514400959, "llm_available": false, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -3952,7 +3954,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 50.58130016550422, + "latency_ms": 34.2594999819994, "llm_available": false, "llm_called": false, "llm_model": null, @@ -3972,7 +3974,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 30.18129989504814, + "latency_ms": 24.028800195083022, "llm_available": false, "llm_called": false, "llm_model": null, @@ -3992,7 +3994,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 361, - "latency_ms": 3566.028299866244, + "latency_ms": 3566.0150999985635, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -4012,7 +4014,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 32.94120007194579, + "latency_ms": 26.04260016232729, "llm_available": false, "llm_called": false, "llm_model": null, @@ -4032,7 +4034,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 34.92819983512163, + "latency_ms": 31.887099845334888, "llm_available": false, "llm_called": false, "llm_model": null, @@ -4052,7 +4054,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 496, - "latency_ms": 3254.0352999195457, + "latency_ms": 3254.028100097552, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -4072,7 +4074,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 32.09630004130304, + "latency_ms": 34.07870000228286, "llm_available": false, "llm_called": false, "llm_model": null, @@ -4092,7 +4094,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 26.938800001516938, + "latency_ms": 24.392900057137012, "llm_available": false, "llm_called": false, "llm_model": null, @@ -4112,7 +4114,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 361, - "latency_ms": 1376.0290002208203, + "latency_ms": 1376.0291999895126, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -4132,7 +4134,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 25.213800137862563, + "latency_ms": 22.934800013899803, "llm_available": false, "llm_called": false, "llm_model": null, @@ -4152,7 +4154,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 29.155299998819828, + "latency_ms": 10.648499941453338, "llm_available": false, "llm_called": false, "llm_model": null, @@ -4172,7 +4174,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 0.02269982360303402, + "latency_ms": 0.014999881386756897, "llm_available": false, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -4192,7 +4194,7 @@ "expected_label": "normal", "fallback_applied": true, "input_tokens": null, - "latency_ms": 26.233799988403916, + "latency_ms": 22.65110006555915, "llm_available": false, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -4212,7 +4214,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 163.8730000704527, + "latency_ms": 22.606899961829185, "llm_available": false, "llm_called": false, "llm_model": null, @@ -4232,7 +4234,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 325, - "latency_ms": 1726.0287999864668, + "latency_ms": 1726.0320000108331, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -4252,7 +4254,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 27.894100174307823, + "latency_ms": 22.20589993521571, "llm_available": false, "llm_called": false, "llm_model": null, @@ -4272,7 +4274,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 25.544699979946017, + "latency_ms": 22.958500077947974, "llm_available": false, "llm_called": false, "llm_model": null, @@ -4292,7 +4294,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 0.022999942302703857, + "latency_ms": 0.024899840354919434, "llm_available": false, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -4312,7 +4314,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 24.406200041994452, + "latency_ms": 24.264799896627665, "llm_available": false, "llm_called": false, "llm_model": null, @@ -4332,7 +4334,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 36.350399954244494, + "latency_ms": 32.09170000627637, "llm_available": false, "llm_called": false, "llm_model": null, @@ -4352,7 +4354,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 496, - "latency_ms": 3224.027900096029, + "latency_ms": 3224.029099872336, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -4372,7 +4374,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 32.97589998692274, + "latency_ms": 31.601499998942018, "llm_available": false, "llm_called": false, "llm_model": null, @@ -4392,7 +4394,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 33.5551998578012, + "latency_ms": 33.170999959111214, "llm_available": false, "llm_called": false, "llm_model": null, @@ -4412,7 +4414,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 496, - "latency_ms": 3654.0360999256372, + "latency_ms": 3654.0838998239487, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -4432,7 +4434,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 41.39239992946386, + "latency_ms": 33.25760015286505, "llm_available": false, "llm_called": false, "llm_model": null, @@ -4452,7 +4454,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 26.280299993231893, + "latency_ms": 25.82149999216199, "llm_available": false, "llm_called": false, "llm_model": null, @@ -4472,7 +4474,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 368, - "latency_ms": 3158.0360999256372, + "latency_ms": 3158.0148001126945, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -4492,7 +4494,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 368, - "latency_ms": 3184.4637998770922, + "latency_ms": 3184.863900013268, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -4512,7 +4514,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 27.390999952331185, + "latency_ms": 23.521999828517437, "llm_available": false, "llm_called": false, "llm_model": null, @@ -4532,7 +4534,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 339, - "latency_ms": 1606.0269999727607, + "latency_ms": 1606.0394000671804, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -4552,7 +4554,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 29.709500027820468, + "latency_ms": 20.93380014412105, "llm_available": false, "llm_called": false, "llm_model": null, @@ -4572,7 +4574,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 45.445100171491504, + "latency_ms": 22.49370003119111, "llm_available": false, "llm_called": false, "llm_model": null, @@ -4592,7 +4594,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 0.0659001525491476, + "latency_ms": 0.025900080800056458, "llm_available": false, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -4612,7 +4614,7 @@ "expected_label": "normal", "fallback_applied": true, "input_tokens": null, - "latency_ms": 35.969800082966685, + "latency_ms": 22.597800008952618, "llm_available": false, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -4632,7 +4634,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 29.804500052705407, + "latency_ms": 19.81660001911223, "llm_available": false, "llm_called": false, "llm_model": null, @@ -4652,7 +4654,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 343, - "latency_ms": 1690.040000071749, + "latency_ms": 1690.0316000077873, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -4672,7 +4674,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 28.307799948379397, + "latency_ms": 21.793500054627657, "llm_available": false, "llm_called": false, "llm_model": null, @@ -4692,7 +4694,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 34.636800177395344, + "latency_ms": 19.44579998962581, "llm_available": false, "llm_called": false, "llm_model": null, @@ -4712,7 +4714,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 0.03269989974796772, + "latency_ms": 0.02259993925690651, "llm_available": false, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -4732,7 +4734,7 @@ "expected_label": "normal", "fallback_applied": true, "input_tokens": null, - "latency_ms": 23.890099953860044, + "latency_ms": 22.907799808308482, "llm_available": false, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -4752,7 +4754,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 26.665500132367015, + "latency_ms": 24.518300080671906, "llm_available": false, "llm_called": false, "llm_model": null, @@ -4772,7 +4774,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 355, - "latency_ms": 3290.0274998601526, + "latency_ms": 3290.017800135538, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -4792,7 +4794,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 30.232900055125356, + "latency_ms": 24.23450001515448, "llm_available": false, "llm_called": false, "llm_model": null, @@ -4812,7 +4814,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 33.72840001247823, + "latency_ms": 26.921600103378296, "llm_available": false, "llm_called": false, "llm_model": null, @@ -4832,7 +4834,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 360, - "latency_ms": 1578.0356999225914, + "latency_ms": 1578.0346000306308, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -4852,7 +4854,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 34.24179996363819, + "latency_ms": 24.016299983486533, "llm_available": false, "llm_called": false, "llm_model": null, @@ -4872,7 +4874,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 25.211500003933907, + "latency_ms": 23.45619979314506, "llm_available": false, "llm_called": false, "llm_model": null, @@ -4892,7 +4894,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 346, - "latency_ms": 1879.0340000260621, + "latency_ms": 1879.0330000184476, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -4912,7 +4914,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 27.122699888423085, + "latency_ms": 21.03750011883676, "llm_available": false, "llm_called": false, "llm_model": null, @@ -4932,7 +4934,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 30.87080013938248, + "latency_ms": 22.18880015425384, "llm_available": false, "llm_called": false, "llm_model": null, @@ -4952,7 +4954,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 0.02359994687139988, + "latency_ms": 0.025999965146183968, "llm_available": false, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -4972,7 +4974,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 26.554000098258257, + "latency_ms": 23.55769998393953, "llm_available": false, "llm_called": false, "llm_model": null, @@ -4992,7 +4994,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 27.728899847716093, + "latency_ms": 25.574899977073073, "llm_available": false, "llm_called": false, "llm_model": null, @@ -5012,7 +5014,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 377, - "latency_ms": 2955.028100097552, + "latency_ms": 2955.0352000351995, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -5032,7 +5034,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 377, - "latency_ms": 2986.1150001361966, + "latency_ms": 2979.5614000596106, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -5052,7 +5054,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 34.92280002683401, + "latency_ms": 24.423500057309866, "llm_available": false, "llm_called": false, "llm_model": null, @@ -5072,7 +5074,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 344, - "latency_ms": 1489.030499882996, + "latency_ms": 1489.0289999879897, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -5092,7 +5094,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 28.043999802321196, + "latency_ms": 23.730999790132046, "llm_available": false, "llm_called": false, "llm_model": null, @@ -5112,7 +5114,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 26.428299956023693, + "latency_ms": 27.966200141236186, "llm_available": false, "llm_called": false, "llm_model": null, @@ -5132,7 +5134,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 0.024900073185563087, + "latency_ms": 0.02429983578622341, "llm_available": false, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -5152,7 +5154,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 28.132700128480792, + "latency_ms": 29.674699995666742, "llm_available": false, "llm_called": false, "llm_model": null, @@ -5172,7 +5174,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 28.64830009639263, + "latency_ms": 26.042900048196316, "llm_available": false, "llm_called": false, "llm_model": null, @@ -5192,7 +5194,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 400, - "latency_ms": 3085.040899962187, + "latency_ms": 3085.029999995604, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -5212,7 +5214,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 28.275599936023355, + "latency_ms": 24.466600036248565, "llm_available": false, "llm_called": false, "llm_model": null, @@ -5232,7 +5234,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 25.899000000208616, + "latency_ms": 24.99469998292625, "llm_available": false, "llm_called": false, "llm_model": null, @@ -5252,7 +5254,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 0.02550007775425911, + "latency_ms": 0.02650008536875248, "llm_available": false, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -5272,7 +5274,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 28.001599945127964, + "latency_ms": 25.447299936786294, "llm_available": false, "llm_called": false, "llm_model": null, @@ -5292,7 +5294,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 21.932000061497092, + "latency_ms": 21.451900014653802, "llm_available": false, "llm_called": false, "llm_model": null, @@ -5312,7 +5314,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 393, - "latency_ms": 3877.028100097552, + "latency_ms": 3877.03210012801, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -5332,7 +5334,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 21.24790009111166, + "latency_ms": 23.66689988411963, "llm_available": false, "llm_called": false, "llm_model": null, @@ -5352,7 +5354,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 27.618099935352802, + "latency_ms": 23.55609997175634, "llm_available": false, "llm_called": false, "llm_model": null, @@ -5372,7 +5374,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 390, - "latency_ms": 3080.0837999396026, + "latency_ms": 3080.0153000000864, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -5392,7 +5394,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 29.95329978875816, + "latency_ms": 23.86320009827614, "llm_available": false, "llm_called": false, "llm_model": null, @@ -5412,7 +5414,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 33.902900060638785, + "latency_ms": 19.10959999077022, "llm_available": false, "llm_called": false, "llm_model": null, @@ -5432,7 +5434,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 390, - "latency_ms": 2872.052700052038, + "latency_ms": 2872.016300007701, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -5452,7 +5454,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 35.19130009226501, + "latency_ms": 22.826100001111627, "llm_available": false, "llm_called": false, "llm_model": null, @@ -5472,7 +5474,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 33.804599894210696, + "latency_ms": 23.84619996882975, "llm_available": false, "llm_called": false, "llm_model": null, @@ -5492,7 +5494,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 394, - "latency_ms": 3324.026199966669, + "latency_ms": 3324.029099872336, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -5512,7 +5514,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 29.805000172927976, + "latency_ms": 32.723700162023306, "llm_available": false, "llm_called": false, "llm_model": null, @@ -5532,7 +5534,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 26.991200167685747, + "latency_ms": 24.719800101593137, "llm_available": false, "llm_called": false, "llm_model": null, @@ -5552,7 +5554,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 384, - "latency_ms": 3750.0278998631984, + "latency_ms": 3750.0305001158267, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -5572,7 +5574,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 32.56290010176599, + "latency_ms": 23.707800079137087, "llm_available": false, "llm_called": false, "llm_model": null, @@ -5592,7 +5594,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 24.228000082075596, + "latency_ms": 23.891899967566133, "llm_available": false, "llm_called": false, "llm_model": null, @@ -5612,7 +5614,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 365, - "latency_ms": 3274.0350000336766, + "latency_ms": 3274.015700003132, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -5632,7 +5634,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 30.498400097712874, + "latency_ms": 25.75210016220808, "llm_available": false, "llm_called": false, "llm_model": null, @@ -5652,7 +5654,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 25.528399972245097, + "latency_ms": 25.56199999526143, "llm_available": false, "llm_called": false, "llm_model": null, @@ -5672,7 +5674,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 370, - "latency_ms": 3427.02760021016, + "latency_ms": 3427.029999995604, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -5692,7 +5694,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 26.89819992519915, + "latency_ms": 24.935900000855327, "llm_available": false, "llm_called": false, "llm_model": null, @@ -5712,7 +5714,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 32.09739993326366, + "latency_ms": 26.182800065726042, "llm_available": false, "llm_called": false, "llm_model": null, @@ -5732,7 +5734,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 393, - "latency_ms": 2948.028599984944, + "latency_ms": 2948.0296998769045, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -5752,7 +5754,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 33.08010008186102, + "latency_ms": 27.21849991939962, "llm_available": false, "llm_called": false, "llm_model": null, @@ -5772,7 +5774,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 32.09719993174076, + "latency_ms": 25.092399911955, "llm_available": false, "llm_called": false, "llm_model": null, @@ -5792,7 +5794,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 383, - "latency_ms": 3004.018799910322, + "latency_ms": 3004.0374999362975, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -5812,7 +5814,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 26.278499979525805, + "latency_ms": 22.215199889615178, "llm_available": false, "llm_called": false, "llm_model": null, @@ -5832,7 +5834,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 31.39309980906546, + "latency_ms": 25.665400084108114, "llm_available": false, "llm_called": false, "llm_model": null, @@ -5852,7 +5854,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 385, - "latency_ms": 3140.0372000504285, + "latency_ms": 3140.0276998616755, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -5872,7 +5874,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 29.97259981930256, + "latency_ms": 25.674100033938885, "llm_available": false, "llm_called": false, "llm_model": null, @@ -5892,7 +5894,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 29.378199949860573, + "latency_ms": 24.72769981250167, "llm_available": false, "llm_called": false, "llm_model": null, @@ -5912,7 +5914,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 0.01610000617802143, + "latency_ms": 0.02450007013976574, "llm_available": false, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -5932,7 +5934,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 31.043499941006303, + "latency_ms": 26.65379992686212, "llm_available": false, "llm_called": false, "llm_model": null, @@ -5952,7 +5954,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 28.784000081941485, + "latency_ms": 25.093100033700466, "llm_available": false, "llm_called": false, "llm_model": null, @@ -5972,7 +5974,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 388, - "latency_ms": 3270.0139998737723, + "latency_ms": 3270.0195000320673, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -5992,7 +5994,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 21.651099901646376, + "latency_ms": 25.202399818226695, "llm_available": false, "llm_called": false, "llm_model": null, @@ -6012,7 +6014,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 28.30630005337298, + "latency_ms": 29.136199969798326, "llm_available": false, "llm_called": false, "llm_model": null, @@ -6032,7 +6034,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 393, - "latency_ms": 3407.018100021407, + "latency_ms": 3407.02869986929, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -6052,7 +6054,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 26.72929991967976, + "latency_ms": 25.605799863114953, "llm_available": false, "llm_called": false, "llm_model": null, @@ -6072,7 +6074,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 27.652400080114603, + "latency_ms": 24.85990012064576, "llm_available": false, "llm_called": false, "llm_model": null, @@ -6092,7 +6094,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 381, - "latency_ms": 3401.03830017522, + "latency_ms": 3401.0293999910355, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -6112,7 +6114,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 27.94139995239675, + "latency_ms": 26.900399941951036, "llm_available": false, "llm_called": false, "llm_model": null, @@ -6132,7 +6134,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 26.03790001012385, + "latency_ms": 22.759499959647655, "llm_available": false, "llm_called": false, "llm_model": null, @@ -6152,7 +6154,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 376, - "latency_ms": 4065.016199890524, + "latency_ms": 4065.0177999027073, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -6172,7 +6174,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 26.982299983501434, + "latency_ms": 24.207700043916702, "llm_available": false, "llm_called": false, "llm_model": null, @@ -6192,7 +6194,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 31.371199991554022, + "latency_ms": 27.240399969741702, "llm_available": false, "llm_called": false, "llm_model": null, @@ -6212,7 +6214,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 385, - "latency_ms": 3441.0348000321537, + "latency_ms": 3441.0308000016958, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -6232,7 +6234,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 27.399900136515498, + "latency_ms": 32.151699997484684, "llm_available": false, "llm_called": false, "llm_model": null, @@ -6252,7 +6254,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 31.203699996694922, + "latency_ms": 27.537199901416898, "llm_available": false, "llm_called": false, "llm_model": null, @@ -6272,7 +6274,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 377, - "latency_ms": 3194.029900111258, + "latency_ms": 3194.0409001950175, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -6292,7 +6294,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 29.026499949395657, + "latency_ms": 26.107899844646454, "llm_available": false, "llm_called": false, "llm_model": null, @@ -6312,7 +6314,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 35.193700110539794, + "latency_ms": 28.86569988913834, "llm_available": false, "llm_called": false, "llm_model": null, @@ -6332,7 +6334,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 404, - "latency_ms": 3096.030499882996, + "latency_ms": 3096.0288002192974, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -6352,7 +6354,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 30.51090007647872, + "latency_ms": 25.71749989874661, "llm_available": false, "llm_called": false, "llm_model": null, @@ -6372,7 +6374,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 37.57379995658994, + "latency_ms": 25.568699929863214, "llm_available": false, "llm_called": false, "llm_model": null, @@ -6392,7 +6394,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 397, - "latency_ms": 2747.0291999895126, + "latency_ms": 2747.034300144762, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -6412,7 +6414,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 39.25159992650151, + "latency_ms": 25.5666000302881, "llm_available": false, "llm_called": false, "llm_model": null, @@ -6432,7 +6434,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 35.468799993395805, + "latency_ms": 25.531399995088577, "llm_available": false, "llm_called": false, "llm_model": null, @@ -6452,7 +6454,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 0.033899908885359764, + "latency_ms": 0.025300076231360435, "llm_available": false, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -6472,7 +6474,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 29.02040001936257, + "latency_ms": 25.310799945145845, "llm_available": false, "llm_called": false, "llm_model": null, @@ -6492,7 +6494,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 28.438600013032556, + "latency_ms": 25.322299916297197, "llm_available": false, "llm_called": false, "llm_model": null, @@ -6512,7 +6514,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 401, - "latency_ms": 3199.012299977243, + "latency_ms": 3199.0293999910355, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -6532,7 +6534,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 27.938100043684244, + "latency_ms": 26.037100004032254, "llm_available": false, "llm_called": false, "llm_model": null, @@ -6552,7 +6554,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 25.608000112697482, + "latency_ms": 25.83350008353591, "llm_available": false, "llm_called": false, "llm_model": null, @@ -6572,7 +6574,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 387, - "latency_ms": 3591.133299967274, + "latency_ms": 3591.0298998784274, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -6592,7 +6594,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 30.708400066941977, + "latency_ms": 29.784500133246183, "llm_available": false, "llm_called": false, "llm_model": null, @@ -6612,7 +6614,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 32.41519979201257, + "latency_ms": 33.419699873775244, "llm_available": false, "llm_called": false, "llm_model": null, @@ -6632,7 +6634,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 397, - "latency_ms": 3340.0171000137925, + "latency_ms": 3340.0173998996615, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -6652,7 +6654,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 23.269099881872535, + "latency_ms": 28.4746999386698, "llm_available": false, "llm_called": false, "llm_model": null, @@ -6672,7 +6674,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 31.253399793058634, + "latency_ms": 27.614100137725472, "llm_available": false, "llm_called": false, "llm_model": null, @@ -6692,7 +6694,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 385, - "latency_ms": 3145.036899931729, + "latency_ms": 3145.034999800846, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -6712,7 +6714,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 31.970799900591373, + "latency_ms": 21.287899930030107, "llm_available": false, "llm_called": false, "llm_model": null, @@ -6732,7 +6734,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 28.288900153711438, + "latency_ms": 20.710099954158068, "llm_available": false, "llm_called": false, "llm_model": null, @@ -6752,7 +6754,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 400, - "latency_ms": 3028.0269000884145, + "latency_ms": 3028.028599984944, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -6772,7 +6774,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 30.069999862462282, + "latency_ms": 25.297299958765507, "llm_available": false, "llm_called": false, "llm_model": null, @@ -6792,7 +6794,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 26.487299939617515, + "latency_ms": 24.363200180232525, "llm_available": false, "llm_called": false, "llm_model": null, @@ -6812,7 +6814,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 385, - "latency_ms": 3541.016899779439, + "latency_ms": 3541.0314998906106, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -6832,7 +6834,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 26.355199981480837, + "latency_ms": 25.218100054189563, "llm_available": false, "llm_called": false, "llm_model": null, @@ -6852,7 +6854,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 28.841400053352118, + "latency_ms": 29.101100051775575, "llm_available": false, "llm_called": false, "llm_model": null, @@ -6872,7 +6874,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 396, - "latency_ms": 2816.020799925551, + "latency_ms": 2816.030600000173, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -6892,7 +6894,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 24.63520015589893, + "latency_ms": 18.363599898293614, "llm_available": false, "llm_called": false, "llm_model": null, @@ -6912,7 +6914,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 28.056900016963482, + "latency_ms": 26.344900019466877, "llm_available": false, "llm_called": false, "llm_model": null, @@ -6932,7 +6934,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 0.024600187316536903, + "latency_ms": 0.011299969628453255, "llm_available": false, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -6952,7 +6954,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 28.396699810400605, + "latency_ms": 25.39149997755885, "llm_available": false, "llm_called": false, "llm_model": null, @@ -6972,7 +6974,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 27.979600010439754, + "latency_ms": 25.219399947673082, "llm_available": false, "llm_called": false, "llm_model": null, @@ -6992,7 +6994,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 392, - "latency_ms": 2898.0287999864668, + "latency_ms": 2898.03039999865, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -7012,7 +7014,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 29.78320000693202, + "latency_ms": 22.467200178653002, "llm_available": false, "llm_called": false, "llm_model": null, @@ -7032,7 +7034,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 31.282000010833144, + "latency_ms": 21.608700044453144, "llm_available": false, "llm_called": false, "llm_model": null, @@ -7052,7 +7054,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 402, - "latency_ms": 3440.0267000868917, + "latency_ms": 3440.03039999865, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -7072,7 +7074,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 28.07500003837049, + "latency_ms": 23.013100028038025, "llm_available": false, "llm_called": false, "llm_model": null, @@ -7092,7 +7094,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 31.653299927711487, + "latency_ms": 22.239399841055274, "llm_available": false, "llm_called": false, "llm_model": null, @@ -7112,7 +7114,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 395, - "latency_ms": 3210.030199997127, + "latency_ms": 3210.0175999011844, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -7132,7 +7134,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 35.09249980561435, + "latency_ms": 24.390000151470304, "llm_available": false, "llm_called": false, "llm_model": null, @@ -7152,7 +7154,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 28.07609993033111, + "latency_ms": 24.24149983562529, "llm_available": false, "llm_called": false, "llm_model": null, @@ -7172,7 +7174,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 396, - "latency_ms": 2931.027800211683, + "latency_ms": 2931.037800054997, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -7192,7 +7194,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 27.08280016668141, + "latency_ms": 27.618899941444397, "llm_available": false, "llm_called": false, "llm_model": null, @@ -7212,7 +7214,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": null, - "latency_ms": 27.17570005916059, + "latency_ms": 23.84530007839203, "llm_available": false, "llm_called": false, "llm_model": null, @@ -7232,7 +7234,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 388, - "latency_ms": 3511.0374999362975, + "latency_ms": 3511.031099887565, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -7252,7 +7254,7 @@ "expected_label": "phishing", "fallback_applied": false, "input_tokens": 388, - "latency_ms": 3538.8847001027316, + "latency_ms": 3537.4950001146644, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -7272,7 +7274,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 24.785999907180667, + "latency_ms": 24.565199855715036, "llm_available": false, "llm_called": false, "llm_model": null, @@ -7292,7 +7294,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 366, - "latency_ms": 2732.0333001371473, + "latency_ms": 2732.0942000187933, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -7312,7 +7314,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 366, - "latency_ms": 2757.651900097728, + "latency_ms": 2757.817299960181, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -7332,7 +7334,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 25.114599848166108, + "latency_ms": 26.8545001745224, "llm_available": false, "llm_called": false, "llm_model": null, @@ -7352,7 +7354,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 360, - "latency_ms": 1691.0360000412911, + "latency_ms": 1691.0297999940813, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -7372,7 +7374,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 360, - "latency_ms": 1715.220900144428, + "latency_ms": 1716.3453999757767, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -7392,7 +7394,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 28.941299999132752, + "latency_ms": 25.681800208985806, "llm_available": false, "llm_called": false, "llm_model": null, @@ -7412,7 +7414,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 0.014800112694501877, + "latency_ms": 0.016200123354792595, "llm_available": false, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -7432,7 +7434,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 34.28729996085167, + "latency_ms": 26.35469986125827, "llm_available": false, "llm_called": false, "llm_model": null, @@ -7452,7 +7454,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 21.752400090917945, + "latency_ms": 20.602300064638257, "llm_available": false, "llm_called": false, "llm_model": null, @@ -7472,7 +7474,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 329, - "latency_ms": 1527.0192001461983, + "latency_ms": 1527.0297999940813, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -7492,7 +7494,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": 329, - "latency_ms": 1548.736399969086, + "latency_ms": 1548.9390001147985, "llm_available": true, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -7512,7 +7514,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 24.032500106841326, + "latency_ms": 24.36889987438917, "llm_available": false, "llm_called": false, "llm_model": null, @@ -7532,7 +7534,7 @@ "expected_label": "normal", "fallback_applied": false, "input_tokens": null, - "latency_ms": 0.025799963623285294, + "latency_ms": 0.02519995905458927, "llm_available": false, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -7552,7 +7554,7 @@ "expected_label": "normal", "fallback_applied": true, "input_tokens": null, - "latency_ms": 30.24750016629696, + "latency_ms": 21.886999951675534, "llm_available": false, "llm_called": true, "llm_model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", @@ -7567,5 +7569,7 @@ } ], "region": "us-east-1", - "source_split": "test" + "source_split": "test", + "split_manifest": "sms_split_v1.csv", + "split_manifest_sha256": "4c33fcc749ca5c8ac4e6b35d19831e3ad8bffb02e72ac4d50d423d7ca19aa5f9" }