Test(#76): Stacking·Claude·Hybrid 평가 기준 통일 및 지표 산출 근거 명확화

data_science/SMSModel/hybrid_evaluation/__init__.py

@@ -4,6 +4,7 @@
     calculate_classification_metrics,
     calculate_cost_metrics,
     calculate_cost_reduction_rate,
+    calculate_full_dataset_metrics,
     calculate_latency_metrics,
     calculate_operational_metrics,
 )
@@ -12,6 +13,7 @@
     CostMetrics,
     EvaluationMode,
     EvaluationRecord,
+    FullDatasetMetrics,
     LatencyMetrics,
     OperationalMetrics,
     OperationalOutcome,
@@ -41,6 +43,7 @@
     "EvaluationMode",
     "EvaluationRecord",
     "EvaluationSample",
+    "FullDatasetMetrics",
     "HybridEvaluationRunner",
     "LatencyMetrics",
     "OperationalMetrics",
@@ -52,6 +55,7 @@
     "calculate_classification_metrics",
     "calculate_cost_metrics",
     "calculate_cost_reduction_rate",
+    "calculate_full_dataset_metrics",
     "calculate_dataset_fingerprint",
     "calculate_latency_metrics",
     "calculate_operational_metrics",

data_science/SMSModel/hybrid_evaluation/metrics.py

@@ -17,13 +17,15 @@
 from data_science.SMSModel.hybrid_evaluation.models import (
     ClassificationMetrics,
     CostMetrics,
+    FullDatasetMetrics,
     LatencyMetrics,
     OperationalMetrics,
     OperationalOutcome,
     TokenUsage,
 )
 
 LABEL_ORDER = ("normal", "phishing")
+UNAVAILABLE_LABEL = "unknown"
 
 def _validate_binary_labels(
     y_true: Sequence[str],
@@ -134,6 +136,86 @@ def calculate_classification_metrics(
         true_positive=true_positive,
     )
 
+
+def calculate_full_dataset_metrics(
+    y_true: Sequence[str],
+    y_pred: Sequence[str],
+) -> FullDatasetMetrics:
+    """UNKNOWN을 실패로 포함한 전체 데이터셋 지표를 계산"""
+
+    true_values = np.asarray(y_true, dtype=str)
+    predicted_values = np.asarray(y_pred, dtype=str)
+
+    if true_values.ndim != 1 or predicted_values.ndim != 1:
+        raise ValueError(
+            "y_true and y_pred must be one-dimensional"
+        )
+
+    if len(true_values) == 0:
+        raise ValueError(
+            "cannot calculate full-dataset metrics from empty labels"
+        )
+
+    if len(true_values) != len(predicted_values):
+        raise ValueError(
+            "y_true and y_pred must have the same length"
+        )
+
+    binary_labels = set(LABEL_ORDER)
+
+    if not set(true_values).issubset(binary_labels):
+        raise ValueError(
+            "y_true contains unsupported labels"
+        )
+
+    allowed_predictions = {*binary_labels, UNAVAILABLE_LABEL}
+
+    if not set(predicted_values).issubset(allowed_predictions):
+        raise ValueError(
+            "y_pred contains unsupported labels"
+        )
+
+    available_mask = np.isin(predicted_values, LABEL_ORDER)
+    unavailable_mask = predicted_values == UNAVAILABLE_LABEL
+    correct_mask = available_mask & (true_values == predicted_values)
+    incorrect_mask = available_mask & (true_values != predicted_values)
+    phishing_mask = true_values == "phishing"
+    detected_phishing_mask = phishing_mask & (
+        predicted_values == "phishing"
+    )
+
+    total_sample_count = len(true_values)
+    available_count = int(available_mask.sum())
+    unavailable_count = int(unavailable_mask.sum())
+    correct_count = int(correct_mask.sum())
+    incorrect_count = int(incorrect_mask.sum())
+    actual_normal_count = int((true_values == "normal").sum())
+    actual_phishing_count = int(phishing_mask.sum())
+    detected_phishing_count = int(detected_phishing_mask.sum())
+    missed_phishing_count = (
+        actual_phishing_count - detected_phishing_count
+    )
+
+    phishing_detection_rate = (
+        detected_phishing_count / actual_phishing_count
+        if actual_phishing_count > 0
+        else 0.0
+    )
+
+    return FullDatasetMetrics(
+        total_sample_count=total_sample_count,
+        available_count=available_count,
+        unavailable_count=unavailable_count,
+        correct_count=correct_count,
+        incorrect_count=incorrect_count,
+        accuracy=correct_count / total_sample_count,
+        actual_normal_count=actual_normal_count,
+        actual_phishing_count=actual_phishing_count,
+        detected_phishing_count=detected_phishing_count,
+        missed_phishing_count=missed_phishing_count,
+        phishing_detection_rate=phishing_detection_rate,
+    )
+
 def calculate_latency_metrics(
         durations_ms: Sequence[float],
 ) -> LatencyMetrics:

data_science/SMSModel/hybrid_evaluation/models.py

@@ -172,3 +172,26 @@ def to_dict(self) -> dict[str, Any]:
         payload["mode"] = self.mode.value
 
         return payload
+
+
+@dataclass(frozen=True)
+class FullDatasetMetrics:
+    """전체 데이터셋의 분류 및 피싱 탐지 성능 지표"""
+
+    total_sample_count: int
+    available_count: int
+    unavailable_count: int
+    correct_count: int
+    incorrect_count: int
+    accuracy: float
+
+    actual_normal_count: int
+    actual_phishing_count: int
+    detected_phishing_count: int
+    missed_phishing_count: int
+    phishing_detection_rate: float
+
+    def to_dict(self) -> dict[str, Any]:
+        """JSON 직렬화가 가능한 dictionary로 변환"""
+
+        return asdict(self)