explainability_engine.py

# # ============================================================
# # 🧠 Streaming Explainability Engine v7.0 (Scope-Aware Upgrade)
# # Error-Aware Attribution • Streaming Drift • Group/Node Conditional
# # ------------------------------------------------------------
# # Major upgrades vs v6.0:
# # ✅ Scope-aware context support
# # ✅ Per-attribute explainability summaries
# # ✅ Per-intersection explainability summaries
# # ✅ Active-scope explainability payloads
# # ✅ Attribute-level explainability drift/debt
# # ✅ scope_snapshot() support
# # ✅ Backward-compatible update contract
# # ============================================================
#
# from __future__ import annotations
#
# import math
# import random
# from collections import defaultdict, deque
# from dataclasses import dataclass
# from itertools import combinations
# from typing import Any, Dict, Optional, List, Tuple, Iterable
#
# import numpy as np
#
#
# # ============================================================
# # 🔧 Helpers
# # ============================================================
#
# def _safe_float(x: Any, default: float = 0.0) -> float:
#     try:
#         v = float(x)
#         if math.isnan(v) or math.isinf(v):
#             return default
#         return v
#     except Exception:
#         return default
#
#
# def _normalize_group(v: Any) -> str:
#     if v is None:
#         return "unknown"
#     try:
#         if isinstance(v, bool):
#             return str(int(v))
#         if isinstance(v, (int, np.integer)):
#             return str(int(v))
#         if isinstance(v, (float, np.floating)):
#             fv = float(v)
#             if math.isfinite(fv) and abs(fv - round(fv)) < 1e-9:
#                 return str(int(round(fv)))
#             return str(fv)
#         s = str(v).strip()
#         return s if s else "unknown"
#     except Exception:
#         return "unknown"
#
#
# def normalize_sensitive(sensitive: Any) -> Tuple[Tuple[str, str], ...]:
#     if sensitive is None:
#         return (("attr", "unknown"),)
#     if isinstance(sensitive, dict):
#         return tuple(sorted((str(k), _normalize_group(v)) for k, v in sensitive.items()))
#     if isinstance(sensitive, (list, tuple)):
#         if len(sensitive) == 0:
#             return (("attr", "unknown"),)
#         if all(isinstance(x, (list, tuple)) and len(x) == 2 for x in sensitive):
#             return tuple(sorted((str(k), _normalize_group(v)) for k, v in sensitive))  # type: ignore
#         return tuple((f"dim{i}", _normalize_group(v)) for i, v in enumerate(sensitive))
#     return (("attr", _normalize_group(sensitive)),)
#
#
# def _sensitive_to_label(s: Tuple[Tuple[str, str], ...]) -> str:
#     try:
#         return "|".join([f"{k}={v}" for k, v in s])
#     except Exception:
#         return str(s)
#
#
# def _stable_topk(items: Iterable[Tuple[str, float]], k: int) -> List[Tuple[str, float]]:
#     return sorted(items, key=lambda kv: (-abs(kv[1]), kv[0]))[:k]
#
#
# def _cosine_drift(a: np.ndarray, b: np.ndarray) -> float:
#     na = float(np.linalg.norm(a))
#     nb = float(np.linalg.norm(b))
#     if na <= 1e-12 or nb <= 1e-12:
#         return 0.0
#     cos = float(np.dot(a, b) / (na * nb))
#     cos = float(np.clip(cos, -1.0, 1.0))
#     return float(1.0 - cos)
#
#
# def _ema_update(old: float, new: float, alpha: float) -> float:
#     return (1.0 - alpha) * old + alpha * new
#
#
# def _normalize_intersection_specs(raw: Any) -> List[Tuple[str, ...]]:
#     out: List[Tuple[str, ...]] = []
#     if not isinstance(raw, list):
#         return out
#     for x in raw:
#         if isinstance(x, (list, tuple)) and len(x) >= 2:
#             out.append(tuple(sorted(str(v) for v in x)))
#     return out
#
#
# def _intersection_name(attrs: Iterable[str]) -> str:
#     return "&".join(sorted(str(a) for a in attrs))
#
#
# def _pair_list_from_sensitive_dict(sens_dict: Dict[str, str], attrs: Iterable[str]) -> Tuple[Tuple[str, str], ...]:
#     items = [(str(a), _normalize_group(sens_dict.get(str(a), "unknown"))) for a in attrs]
#     return tuple(sorted(items))
#
#
# # ============================================================
# # 📦 Data classes
# # ============================================================
#
# @dataclass
# class AttributionStats:
#     ema: float = 0.0
#     count: int = 0
#
#
# # ============================================================
# # 🧠 StreamingExplainabilityEngine
# # ============================================================
#
# class StreamingExplainabilityEngine:
#     """
#     Streaming explainability with online drift detection.
#
#     Key outputs (update return):
#       - local_importance
#       - global_drift
#       - distribution_drift
#       - top_features
#       - attribute_metrics
#       - intersection_metrics
#       - active_scope
#     """
#
#     def __init__(
#         self,
#         window_size: int = 500,
#         top_k: int = 10,
#         ema_alpha: float = 0.05,
#
#         # compute controls
#         max_features: int = 2000,
#         check_features_per_update: int = 30,
#         update_every: int = 1,
#         sample_features: bool = True,
#         seed: int = 42,
#
#         # drift controls
#         drift_min_history: int = 50,
#         drift_chunk_min: int = 20,
#
#         # baseline imputation
#         impute_strategy: str = "mean",  # mean | zero
#
#         # scope controls
#         max_intersections: int = 16,
#     ):
#         self.window_size = int(max(10, window_size))
#         self.top_k = int(max(1, top_k))
#         self.ema_alpha = float(np.clip(ema_alpha, 0.001, 0.5))
#
#         self.max_features = int(max(50, max_features))
#         self.check_features_per_update = int(max(1, check_features_per_update))
#         self.update_every = int(max(1, update_every))
#         self.sample_features = bool(sample_features)
#         self.rng = random.Random(int(seed))
#
#         self.drift_min_history = int(max(20, drift_min_history))
#         self.drift_chunk_min = int(max(10, drift_chunk_min))
#
#         self.impute_strategy = str(impute_strategy).lower()
#         if self.impute_strategy not in {"mean", "zero"}:
#             self.impute_strategy = "mean"
#
#         self.max_intersections = int(max(1, max_intersections))
#
#         # feature vocabulary
#         self.known_features: List[str] = []
#         self._known_set = set()
#
#         # running means of raw feature values
#         self.feature_val_mean = defaultdict(float)
#         self.feature_val_count = defaultdict(int)
#
#         # global stats
#         self.feature_stats = defaultdict(AttributionStats)
#         self.feature_history = defaultdict(lambda: deque(maxlen=self.window_size))
#
#         # global aligned vector histories
#         self._vector_history = deque(maxlen=self.window_size)
#         self._abs_attr_vector_history = deque(maxlen=self.window_size)
#
#         # global group/node views
#         self.group_stats = defaultdict(lambda: defaultdict(AttributionStats))
#         self.node_stats = defaultdict(lambda: defaultdict(AttributionStats))
#
#         # per-attribute / per-intersection explainability views
#         self.attribute_stats = defaultdict(lambda: defaultdict(AttributionStats))      # attr -> feature -> stats
#         self.attribute_group_stats = defaultdict(lambda: defaultdict(lambda: defaultdict(AttributionStats)))  # attr -> group -> feature -> stats
#         self.intersection_stats = defaultdict(lambda: defaultdict(AttributionStats))   # inter_name -> feature -> stats
#         self.intersection_group_stats = defaultdict(lambda: defaultdict(lambda: defaultdict(AttributionStats)))  # inter_name -> group -> feature -> stats
#
#         # aligned drift histories per attribute / intersection
#         self.attribute_vector_history = defaultdict(lambda: deque(maxlen=self.window_size))
#         self.attribute_abs_vector_history = defaultdict(lambda: deque(maxlen=self.window_size))
#         self.intersection_vector_history = defaultdict(lambda: deque(maxlen=self.window_size))
#         self.intersection_abs_vector_history = defaultdict(lambda: deque(maxlen=self.window_size))
#
#         # scope context
#         self.scope_state: Dict[str, Any] = {}
#         self.fairness_scope_snapshot: Dict[str, Any] = {}
#         self.active_sensitive_dict: Dict[str, Any] = {}
#
#         self._t = 0
#
#     # ============================================================
#     # Scope helpers
#     # ============================================================
#
#     def update_scope_context(
#         self,
#         scope_state: Optional[Dict[str, Any]] = None,
#         fairness_scope_snapshot: Optional[Dict[str, Any]] = None,
#         active_sensitive_dict: Optional[Dict[str, Any]] = None,
#     ) -> None:
#         if isinstance(scope_state, dict):
#             self.scope_state = dict(scope_state)
#         if isinstance(fairness_scope_snapshot, dict):
#             self.fairness_scope_snapshot = dict(fairness_scope_snapshot)
#         if isinstance(active_sensitive_dict, dict):
#             self.active_sensitive_dict = dict(active_sensitive_dict)
#
#     def _active_scope_attributes(self, fallback_sensitive: Optional[Dict[str, str]] = None) -> List[str]:
#         attrs = []
#         if isinstance(self.scope_state, dict):
#             attrs = self.scope_state.get("active_attributes", [])
#         if not isinstance(attrs, list) or not attrs:
#             if isinstance(fallback_sensitive, dict) and fallback_sensitive:
#                 attrs = list(sorted(fallback_sensitive.keys()))
#         return [str(a) for a in attrs]
#
#     def _active_scope_intersections(self, fallback_sensitive: Optional[Dict[str, str]] = None) -> List[Tuple[str, ...]]:
#         raw = []
#         if isinstance(self.scope_state, dict):
#             raw = self.scope_state.get("active_intersections", [])
#         out = _normalize_intersection_specs(raw)
#         if out:
#             return out[: self.max_intersections]
#
#         if isinstance(fallback_sensitive, dict):
#             keys = sorted(fallback_sensitive.keys())
#             combos = []
#             if len(keys) >= 2:
#                 for c in combinations(keys, 2):
#                     combos.append(tuple(c))
#                     if len(combos) >= self.max_intersections:
#                         break
#             return combos
#         return []
#
#     def _primary_scope_attribute(self, fallback_sensitive: Optional[Dict[str, str]] = None) -> Optional[str]:
#         if isinstance(self.scope_state, dict):
#             p = self.scope_state.get("primary_attribute")
#             if p is not None:
#                 return str(p)
#         attrs = self._active_scope_attributes(fallback_sensitive=fallback_sensitive)
#         return attrs[0] if attrs else None
#
#     def _active_scope_payload(self) -> Dict[str, Any]:
#         attrs = self._active_scope_attributes()
#         inters = self._active_scope_intersections()
#         primary = self._primary_scope_attribute()
#
#         attribute_metrics = self._attribute_metrics_payload()
#         active_attr_metrics = {a: attribute_metrics.get(a, {}) for a in attrs}
#         attr_debt = {
#             a: float(attribute_metrics.get(a, {}).get("explainability_debt", 0.0))
#             for a in attrs
#         }
#
#         return {
#             "mode": self.scope_state.get("mode", "unknown") if isinstance(self.scope_state, dict) else "unknown",
#             "primary_attribute": primary,
#             "active_attributes": attrs,
#             "active_intersections": inters,
#             "active_attribute_metrics": active_attr_metrics,
#             "active_attribute_debt": attr_debt,
#             "scope_snapshot_profiles": self.fairness_scope_snapshot.get("profiles", {})
#             if isinstance(self.fairness_scope_snapshot, dict)
#             else {},
#         }
#
#     # ============================================================
#     # Prediction helper
#     # ============================================================
#
#     def _predict_score(self, model: Any, x: Dict[str, Any], y_true: Optional[Any] = None) -> Tuple[float, Optional[Any]]:
#         try:
#             proba = model.predict_proba_one(x)
#         except Exception:
#             proba = None
#
#         if isinstance(proba, dict) and proba:
#             if 1 in proba or True in proba:
#                 s = _safe_float(proba.get(1, proba.get(True, 0.0)), 0.0)
#                 s = float(np.clip(s, 0.0, 1.0))
#                 yhat = max(proba, key=proba.get)
#                 return s, yhat
#
#             if y_true is not None and y_true in proba:
#                 s = _safe_float(proba.get(y_true, 0.0), 0.0)
#                 s = float(np.clip(s, 0.0, 1.0))
#                 yhat = max(proba, key=proba.get)
#                 return s, yhat
#
#             yhat = max(proba, key=proba.get)
#             s = _safe_float(proba.get(yhat, 0.0), 0.0)
#             s = float(np.clip(s, 0.0, 1.0))
#             return s, yhat
#
#         try:
#             yhat = model.predict_one(x)
#         except Exception:
#             yhat = None
#
#         if y_true is not None and yhat is not None:
#             return (1.0 if yhat == y_true else 0.0), yhat
#
#         return 0.0, yhat
#
#     # ============================================================
#     # Vocabulary + imputation
#     # ============================================================
#
#     def _update_vocab_and_means(self, x: Dict[str, Any]) -> None:
#         for f, v in x.items():
#             if f not in self._known_set:
#                 if len(self.known_features) >= self.max_features:
#                     continue
#                 self._known_set.add(f)
#                 self.known_features.append(f)
#
#             val = _safe_float(v, default=None)  # type: ignore
#             if val is None:
#                 continue
#             c = self.feature_val_count[f]
#             self.feature_val_count[f] = c + 1
#             self.feature_val_mean[f] = self.feature_val_mean[f] + (val - self.feature_val_mean[f]) / (c + 1)
#
#     def _impute(self, feature: str) -> float:
#         if self.impute_strategy == "zero":
#             return 0.0
#         return float(self.feature_val_mean.get(feature, 0.0))
#
#     # ============================================================
#     # MAIN UPDATE
#     # ============================================================
#
#     def update(
#         self,
#         model: Any,
#         x: Dict[str, Any],
#         y_true: Optional[Any] = None,
#         sensitive: Optional[Any] = None,
#         node_id: Optional[Any] = None,
#         scope_state: Optional[Dict[str, Any]] = None,
#         fairness_scope_snapshot: Optional[Dict[str, Any]] = None,
#         active_sensitive_dict: Optional[Dict[str, Any]] = None,
#     ) -> Dict[str, Any]:
#         self._t += 1
#
#         if scope_state is not None or fairness_scope_snapshot is not None or active_sensitive_dict is not None:
#             self.update_scope_context(
#                 scope_state=scope_state,
#                 fairness_scope_snapshot=fairness_scope_snapshot,
#                 active_sensitive_dict=active_sensitive_dict,
#             )
#
#         do_update = (self._t % self.update_every == 0)
#
#         try:
#             self._update_vocab_and_means(x)
#         except Exception:
#             pass
#
#         sens_norm = normalize_sensitive(sensitive) if sensitive is not None else (("attr", "unknown"),)
#         sens_dict = {k: v for k, v in sens_norm}
#
#         if not do_update:
#             return {
#                 "local_importance": {},
#                 "global_drift": float(self._compute_global_drift()),
#                 "distribution_drift": float(self._compute_distribution_drift()),
#                 "top_features": _stable_topk(((k, v.ema) for k, v in self.feature_stats.items()), self.top_k),
#                 "attribute_metrics": self._attribute_metrics_payload(),
#                 "intersection_metrics": self._intersection_metrics_payload(),
#                 "active_scope": self._active_scope_payload(),
#             }
#
#         base_score, _ = self._predict_score(model, x, y_true=y_true)
#
#         err_w = 1.0
#         if y_true is not None:
#             yt = 1.0 if (y_true == 1 or y_true is True) else 0.0
#             err_w = abs(yt - float(base_score))
#         err_w = float(np.clip(err_w, 0.0, 1.0))
#
#         feats = list(self.known_features)
#         if not feats:
#             return {
#                 "local_importance": {},
#                 "global_drift": 0.0,
#                 "distribution_drift": 0.0,
#                 "top_features": [],
#                 "attribute_metrics": {},
#                 "intersection_metrics": {},
#                 "active_scope": self._active_scope_payload(),
#             }
#
#         if len(feats) > self.check_features_per_update:
#             if self.sample_features:
#                 feats_to_check = self.rng.sample(feats, self.check_features_per_update)
#             else:
#                 feats_to_check = feats[: self.check_features_per_update]
#         else:
#             feats_to_check = feats
#
#         active_attrs = self._active_scope_attributes(fallback_sensitive=sens_dict)
#         active_inters = self._active_scope_intersections(fallback_sensitive=sens_dict)
#
#         local_importance: Dict[str, float] = {}
#
#         for f in feats_to_check:
#             if f not in x:
#                 continue
#             try:
#                 x_mod = dict(x)
#                 x_mod[f] = self._impute(f)
#
#                 mod_score, _ = self._predict_score(model, x_mod, y_true=y_true)
#                 contrib = (float(base_score) - float(mod_score)) * err_w
#                 contrib = _safe_float(contrib, 0.0)
#                 local_importance[f] = contrib
#
#                 # global
#                 self.feature_history[f].append(contrib)
#                 hist_mean = float(np.mean(self.feature_history[f])) if self.feature_history[f] else contrib
#
#                 st = self.feature_stats[f]
#                 st.ema = _ema_update(st.ema, hist_mean, self.ema_alpha)
#                 st.count += 1
#
#                 # full-group / node
#                 if sensitive is not None:
#                     g = _sensitive_to_label(sens_norm)
#                     gst = self.group_stats[g][f]
#                     gst.ema = _ema_update(gst.ema, contrib, self.ema_alpha)
#                     gst.count += 1
#
#                 if node_id is not None:
#                     nid = str(node_id)
#                     nst = self.node_stats[nid][f]
#                     nst.ema = _ema_update(nst.ema, contrib, self.ema_alpha)
#                     nst.count += 1
#
#                 # per-attribute
#                 for attr in active_attrs:
#                     ast = self.attribute_stats[attr][f]
#                     ast.ema = _ema_update(ast.ema, contrib, self.ema_alpha)
#                     ast.count += 1
#
#                     attr_group = str(sens_dict.get(attr, "unknown"))
#                     agst = self.attribute_group_stats[attr][attr_group][f]
#                     agst.ema = _ema_update(agst.ema, contrib, self.ema_alpha)
#                     agst.count += 1
#
#                 # per-intersection
#                 for inter in active_inters:
#                     inter_name = _intersection_name(inter)
#                     ist = self.intersection_stats[inter_name][f]
#                     ist.ema = _ema_update(ist.ema, contrib, self.ema_alpha)
#                     ist.count += 1
#
#                     inter_group = _sensitive_to_label(_pair_list_from_sensitive_dict(sens_dict, inter))
#                     igst = self.intersection_group_stats[inter_name][inter_group][f]
#                     igst.ema = _ema_update(igst.ema, contrib, self.ema_alpha)
#                     igst.count += 1
#
#             except Exception:
#                 continue
#
#         # global aligned histories
#         top_global = _stable_topk(((k, v.ema) for k, v in self.feature_stats.items()), self.top_k)
#         aligned_feats = [k for k, _ in top_global]
#         if aligned_feats:
#             vec_map = {f: float(self.feature_stats[f].ema) for f in aligned_feats}
#             abs_map = {f: float(abs(self.feature_stats[f].ema)) for f in aligned_feats}
#             self._vector_history.append(vec_map)
#             self._abs_attr_vector_history.append(abs_map)
#
#         # per-attribute aligned histories
#         for attr in active_attrs:
#             top_attr = _stable_topk(((k, v.ema) for k, v in self.attribute_stats[attr].items()), self.top_k)
#             attr_feats = [k for k, _ in top_attr]
#             if attr_feats:
#                 vec_map = {f: float(self.attribute_stats[attr][f].ema) for f in attr_feats}
#                 abs_map = {f: float(abs(self.attribute_stats[attr][f].ema)) for f in attr_feats}
#                 self.attribute_vector_history[attr].append(vec_map)
#                 self.attribute_abs_vector_history[attr].append(abs_map)
#
#         # per-intersection aligned histories
#         for inter in active_inters:
#             inter_name = _intersection_name(inter)
#             top_inter = _stable_topk(((k, v.ema) for k, v in self.intersection_stats[inter_name].items()), self.top_k)
#             inter_feats = [k for k, _ in top_inter]
#             if inter_feats:
#                 vec_map = {f: float(self.intersection_stats[inter_name][f].ema) for f in inter_feats}
#                 abs_map = {f: float(abs(self.intersection_stats[inter_name][f].ema)) for f in inter_feats}
#                 self.intersection_vector_history[inter_name].append(vec_map)
#                 self.intersection_abs_vector_history[inter_name].append(abs_map)
#
#         return {
#             "local_importance": local_importance,
#             "global_drift": float(self._compute_global_drift()),
#             "distribution_drift": float(self._compute_distribution_drift()),
#             "top_features": top_global,
#             "attribute_metrics": self._attribute_metrics_payload(),
#             "intersection_metrics": self._intersection_metrics_payload(),
#             "active_scope": self._active_scope_payload(),
#         }
#
#     # ============================================================
#     # Drift computations
#     # ============================================================
#
#     def _compute_vector_drift(self, history: deque) -> float:
#         if len(history) < self.drift_min_history:
#             return 0.0
#         hist = list(history)
#         if len(hist) < 2 * self.drift_chunk_min:
#             return 0.0
#         mid = len(hist) // 2
#         old_maps = hist[:mid]
#         new_maps = hist[mid:]
#
#         feat_set = set()
#         for m in old_maps:
#             feat_set.update(m.keys())
#         for m in new_maps:
#             feat_set.update(m.keys())
#         if not feat_set:
#             return 0.0
#         feats = sorted(feat_set)
#
#         old_arr = np.array([[float(m.get(f, 0.0)) for f in feats] for m in old_maps], dtype=float)
#         new_arr = np.array([[float(m.get(f, 0.0)) for f in feats] for m in new_maps], dtype=float)
#         old = np.mean(old_arr, axis=0)
#         new = np.mean(new_arr, axis=0)
#         return _cosine_drift(old, new)
#
#     def _compute_abs_distribution_drift(self, history: deque) -> float:
#         if len(history) < self.drift_min_history:
#             return 0.0
#         hist = list(history)
#         if len(hist) < 2 * self.drift_chunk_min:
#             return 0.0
#         mid = len(hist) // 2
#         old_maps = hist[:mid]
#         new_maps = hist[mid:]
#
#         feat_set = set()
#         for m in old_maps:
#             feat_set.update(m.keys())
#         for m in new_maps:
#             feat_set.update(m.keys())
#         if not feat_set:
#             return 0.0
#         feats = sorted(feat_set)
#
#         p = np.mean(np.array([[float(m.get(f, 0.0)) for f in feats] for m in old_maps], dtype=float), axis=0)
#         q = np.mean(np.array([[float(m.get(f, 0.0)) for f in feats] for m in new_maps], dtype=float), axis=0)
#
#         p = np.maximum(p, 1e-12)
#         q = np.maximum(q, 1e-12)
#         p = p / np.sum(p)
#         q = q / np.sum(q)
#
#         kl_pq = float(np.sum(p * np.log(p / q)))
#         kl_qp = float(np.sum(q * np.log(q / p)))
#         val = 0.5 * (kl_pq + kl_qp)
#         return float(np.tanh(val))
#
#     def _compute_global_drift(self) -> float:
#         return self._compute_vector_drift(self._vector_history)
#
#     def _compute_distribution_drift(self) -> float:
#         return self._compute_abs_distribution_drift(self._abs_attr_vector_history)
#
#     def _compute_attribute_drift(self, attr: str) -> float:
#         return self._compute_vector_drift(self.attribute_vector_history.get(attr, deque()))
#
#     def _compute_attribute_distribution_drift(self, attr: str) -> float:
#         return self._compute_abs_distribution_drift(self.attribute_abs_vector_history.get(attr, deque()))
#
#     def _compute_intersection_drift(self, inter_name: str) -> float:
#         return self._compute_vector_drift(self.intersection_vector_history.get(inter_name, deque()))
#
#     def _compute_intersection_distribution_drift(self, inter_name: str) -> float:
#         return self._compute_abs_distribution_drift(self.intersection_abs_vector_history.get(inter_name, deque()))
#
#     # ============================================================
#     # Node & group inspection APIs
#     # ============================================================
#
#     def get_node_importance(self, node_id: Any, top_k: Optional[int] = None) -> List[Tuple[str, float]]:
#         nid = str(node_id)
#         if nid not in self.node_stats:
#             return []
#         k = self.top_k if top_k is None else int(max(1, top_k))
#         return _stable_topk(((f, st.ema) for f, st in self.node_stats[nid].items()), k)
#
#     def get_group_importance(self, sensitive_group: Any, top_k: Optional[int] = None) -> List[Tuple[str, float]]:
#         g = _normalize_group(sensitive_group)
#         if g not in self.group_stats:
#             return []
#         k = self.top_k if top_k is None else int(max(1, top_k))
#         return _stable_topk(((f, st.ema) for f, st in self.group_stats[g].items()), k)
#
#     def get_attribute_importance(self, attr: str, top_k: Optional[int] = None) -> List[Tuple[str, float]]:
#         if attr not in self.attribute_stats:
#             return []
#         k = self.top_k if top_k is None else int(max(1, top_k))
#         return _stable_topk(((f, st.ema) for f, st in self.attribute_stats[attr].items()), k)
#
#     def get_attribute_group_importance(self, attr: str, group: Any, top_k: Optional[int] = None) -> List[Tuple[str, float]]:
#         g = _normalize_group(group)
#         if attr not in self.attribute_group_stats or g not in self.attribute_group_stats[attr]:
#             return []
#         k = self.top_k if top_k is None else int(max(1, top_k))
#         return _stable_topk(((f, st.ema) for f, st in self.attribute_group_stats[attr][g].items()), k)
#
#     def get_intersection_importance(self, inter_name: str, top_k: Optional[int] = None) -> List[Tuple[str, float]]:
#         if inter_name not in self.intersection_stats:
#             return []
#         k = self.top_k if top_k is None else int(max(1, top_k))
#         return _stable_topk(((f, st.ema) for f, st in self.intersection_stats[inter_name].items()), k)
#
#     # ============================================================
#     # Scope payloads
#     # ============================================================
#
#     def _attribute_metrics_payload(self) -> Dict[str, Dict[str, Any]]:
#         attrs = set(self.attribute_stats.keys())
#         active_attrs = set(self._active_scope_attributes())
#         primary_attr = self._primary_scope_attribute()
#
#         out: Dict[str, Dict[str, Any]] = {}
#         for attr in sorted(attrs):
#             top_features = self.get_attribute_importance(attr, top_k=self.top_k)
#
#             # group separation proxy: range of absolute feature attribution means across groups
#             group_maps = self.attribute_group_stats.get(attr, {})
#             group_separation = 0.0
#             if group_maps:
#                 feat_set = set()
#                 for gm in group_maps.values():
#                     feat_set.update(gm.keys())
#                 for feat in feat_set:
#                     vals = [abs(float(gm.get(feat, AttributionStats()).ema)) for gm in group_maps.values()]
#                     if len(vals) >= 2:
#                         group_separation = max(group_separation, float(max(vals) - min(vals)))
#
#             global_vals = np.array([abs(float(v)) for _, v in top_features], dtype=float) if top_features else np.array([], dtype=float)
#             explainability_strength = float(np.mean(global_vals)) if global_vals.size else 0.0
#             drift = float(self._compute_attribute_drift(attr))
#             dist_drift = float(self._compute_attribute_distribution_drift(attr))
#
#             out[attr] = {
#                 "top_features": top_features,
#                 "drift": drift,
#                 "distribution_drift": dist_drift,
#                 "group_separation": float(group_separation),
#                 "explainability_strength": float(explainability_strength),
#                 "explainability_debt": float(drift + dist_drift),
#                 "groups_tracked": int(len(group_maps)),
#                 "is_active": bool(attr in active_attrs),
#                 "is_primary": bool(attr == primary_attr),
#                 "scope_profile": self.fairness_scope_snapshot.get("profiles", {}).get(attr, {})
#                 if isinstance(self.fairness_scope_snapshot, dict)
#                 else {},
#             }
#
#         return out
#
#     def _intersection_metrics_payload(self) -> Dict[str, Dict[str, Any]]:
#         inters = set(self.intersection_stats.keys())
#         active_inter_names = {_intersection_name(x) for x in self._active_scope_intersections()}
#
#         out: Dict[str, Dict[str, Any]] = {}
#         for inter_name in sorted(inters):
#             top_features = self.get_intersection_importance(inter_name, top_k=self.top_k)
#             group_maps = self.intersection_group_stats.get(inter_name, {})
#
#             group_separation = 0.0
#             if group_maps:
#                 feat_set = set()
#                 for gm in group_maps.values():
#                     feat_set.update(gm.keys())
#                 for feat in feat_set:
#                     vals = [abs(float(gm.get(feat, AttributionStats()).ema)) for gm in group_maps.values()]
#                     if len(vals) >= 2:
#                         group_separation = max(group_separation, float(max(vals) - min(vals)))
#
#             global_vals = np.array([abs(float(v)) for _, v in top_features], dtype=float) if top_features else np.array([], dtype=float)
#             explainability_strength = float(np.mean(global_vals)) if global_vals.size else 0.0
#
#             out[inter_name] = {
#                 "top_features": top_features,
#                 "drift": float(self._compute_intersection_drift(inter_name)),
#                 "distribution_drift": float(self._compute_intersection_distribution_drift(inter_name)),
#                 "group_separation": float(group_separation),
#                 "explainability_strength": float(explainability_strength),
#                 "groups_tracked": int(len(group_maps)),
#                 "is_active": bool(inter_name in active_inter_names),
#             }
#
#         return out
#
#     def scope_snapshot(self) -> Dict[str, Any]:
#         return {
#             "scope_state": dict(self.scope_state) if isinstance(self.scope_state, dict) else {},
#             "fairness_scope_snapshot": dict(self.fairness_scope_snapshot)
#             if isinstance(self.fairness_scope_snapshot, dict)
#             else {},
#             "active_scope": self._active_scope_payload(),
#             "attribute_metrics": self._attribute_metrics_payload(),
#             "intersection_metrics": self._intersection_metrics_payload(),
#         }
#
#     def snapshot(self) -> Dict[str, Any]:
#         attribute_metrics = self._attribute_metrics_payload()
#         intersection_metrics = self._intersection_metrics_payload()
#         active_scope = self._active_scope_payload()
#
#         return {
#             "timestep": int(self._t),
#             "global_drift": float(self._compute_global_drift()),
#             "distribution_drift": float(self._compute_distribution_drift()),
#             "top_features": _stable_topk(((k, v.ema) for k, v in self.feature_stats.items()), self.top_k),
#             "attribute_metrics": attribute_metrics,
#             "intersection_metrics": intersection_metrics,
#             "attribute_debt": {
#                 k: float(v.get("explainability_debt", 0.0))
#                 for k, v in attribute_metrics.items()
#             },
#             "active_scope": active_scope,
#             "scope_state": dict(self.scope_state) if isinstance(self.scope_state, dict) else {},
#             "fairness_scope_snapshot": dict(self.fairness_scope_snapshot)
#             if isinstance(self.fairness_scope_snapshot, dict)
#             else {},
#             "scope_snapshot": self.scope_snapshot(),
#             "known_feature_count": int(len(self.known_features)),
#         }
#
#     def reset(self) -> None:
#         self.known_features.clear()
#         self._known_set.clear()
#
#         self.feature_val_mean.clear()
#         self.feature_val_count.clear()
#
#         self.feature_stats.clear()
#         self.feature_history.clear()
#
#         self.group_stats.clear()
#         self.node_stats.clear()
#
#         self.attribute_stats.clear()
#         self.attribute_group_stats.clear()
#         self.intersection_stats.clear()
#         self.intersection_group_stats.clear()
#
#         self._vector_history.clear()
#         self._abs_attr_vector_history.clear()
#         self.attribute_vector_history.clear()
#         self.attribute_abs_vector_history.clear()
#         self.intersection_vector_history.clear()
#         self.intersection_abs_vector_history.clear()
#
#         self.scope_state = {}
#         self.fairness_scope_snapshot = {}
#         self.active_sensitive_dict = {}
#
#         self._t = 0

# ============================================================
# 🧠 Streaming Explainability Engine v7.0 (Scope-Aware Upgrade)
# Error-Aware Attribution • Streaming Drift • Group/Node Conditional
# ------------------------------------------------------------
# Major upgrades vs v6.0:
# ✅ Scope-aware context support
# ✅ Per-attribute explainability summaries
# ✅ Per-intersection explainability summaries
# ✅ Active-scope explainability payloads
# ✅ Attribute-level explainability drift/debt
# ✅ scope_snapshot() support
# ✅ Backward-compatible update contract
# ============================================================

from __future__ import annotations

import math
import random
from collections import defaultdict, deque
from dataclasses import dataclass
from itertools import combinations
from typing import Any, Dict, Optional, List, Tuple, Iterable

import numpy as np


# ============================================================
# 🔧 Helpers
# ============================================================

def _safe_float(x: Any, default: float = 0.0) -> float:
    try:
        v = float(x)
        if math.isnan(v) or math.isinf(v):
            return default
        return v
    except Exception:
        return default


def _normalize_group(v: Any) -> str:
    if v is None:
        return "unknown"
    try:
        if isinstance(v, bool):
            return str(int(v))
        if isinstance(v, (int, np.integer)):
            return str(int(v))
        if isinstance(v, (float, np.floating)):
            fv = float(v)
            if math.isfinite(fv) and abs(fv - round(fv)) < 1e-9:
                return str(int(round(fv)))
            return str(fv)
        s = str(v).strip()
        return s if s else "unknown"
    except Exception:
        return "unknown"


def normalize_sensitive(sensitive: Any) -> Tuple[Tuple[str, str], ...]:
    if sensitive is None:
        return (("attr", "unknown"),)
    if isinstance(sensitive, dict):
        return tuple(sorted((str(k), _normalize_group(v)) for k, v in sensitive.items()))
    if isinstance(sensitive, (list, tuple)):
        if len(sensitive) == 0:
            return (("attr", "unknown"),)
        if all(isinstance(x, (list, tuple)) and len(x) == 2 for x in sensitive):
            return tuple(sorted((str(k), _normalize_group(v)) for k, v in sensitive))  # type: ignore
        return tuple((f"dim{i}", _normalize_group(v)) for i, v in enumerate(sensitive))
    return (("attr", _normalize_group(sensitive)),)


def _sensitive_to_label(s: Tuple[Tuple[str, str], ...]) -> str:
    try:
        return "|".join([f"{k}={v}" for k, v in s])
    except Exception:
        return str(s)


def _stable_topk(items: Iterable[Tuple[str, float]], k: int) -> List[Tuple[str, float]]:
    return sorted(items, key=lambda kv: (-abs(kv[1]), kv[0]))[:k]


def _cosine_drift(a: np.ndarray, b: np.ndarray) -> float:
    na = float(np.linalg.norm(a))
    nb = float(np.linalg.norm(b))
    if na <= 1e-12 or nb <= 1e-12:
        return 0.0
    cos = float(np.dot(a, b) / (na * nb))
    cos = float(np.clip(cos, -1.0, 1.0))
    return float(1.0 - cos)


def _ema_update(old: float, new: float, alpha: float) -> float:
    return (1.0 - alpha) * old + alpha * new


def _normalize_intersection_specs(raw: Any) -> List[Tuple[str, ...]]:
    out: List[Tuple[str, ...]] = []
    if not isinstance(raw, list):
        return out
    for x in raw:
        if isinstance(x, (list, tuple)) and len(x) >= 2:
            out.append(tuple(sorted(str(v) for v in x)))
    return out


def _intersection_name(attrs: Iterable[str]) -> str:
    return "&".join(sorted(str(a) for a in attrs))


def _pair_list_from_sensitive_dict(sens_dict: Dict[str, str], attrs: Iterable[str]) -> Tuple[Tuple[str, str], ...]:
    items = [(str(a), _normalize_group(sens_dict.get(str(a), "unknown"))) for a in attrs]
    return tuple(sorted(items))


# ============================================================
# 📦 Data classes
# ============================================================

@dataclass
class AttributionStats:
    ema: float = 0.0
    count: int = 0


# ============================================================
# 🧠 StreamingExplainabilityEngine
# ============================================================

class StreamingExplainabilityEngine:
    """
    Streaming explainability with online drift detection.

    Key outputs (update return):
      - local_importance
      - global_drift
      - distribution_drift
      - top_features
      - attribute_metrics
      - intersection_metrics
      - active_scope
    """

    def __init__(
        self,
        window_size: int = 500,
        top_k: int = 10,
        ema_alpha: float = 0.05,

        # compute controls
        max_features: int = 2000,
        check_features_per_update: int = 30,
        update_every: int = 1,
        sample_features: bool = True,
        seed: int = 42,

        # drift controls
        drift_min_history: int = 50,
        drift_chunk_min: int = 20,

        # baseline imputation
        impute_strategy: str = "mean",  # mean | zero

        # scope controls
        max_intersections: int = 16,
    ):
        self.window_size = int(max(10, window_size))
        self.top_k = int(max(1, top_k))
        self.ema_alpha = float(np.clip(ema_alpha, 0.001, 0.5))

        self.max_features = int(max(50, max_features))
        self.check_features_per_update = int(max(1, check_features_per_update))
        self.update_every = int(max(1, update_every))
        self.sample_features = bool(sample_features)
        self.rng = random.Random(int(seed))

        self.drift_min_history = int(max(20, drift_min_history))
        self.drift_chunk_min = int(max(10, drift_chunk_min))

        self.impute_strategy = str(impute_strategy).lower()
        if self.impute_strategy not in {"mean", "zero"}:
            self.impute_strategy = "mean"

        self.max_intersections = int(max(1, max_intersections))

        # feature vocabulary
        self.known_features: List[str] = []
        self._known_set = set()

        # running means of raw feature values
        self.feature_val_mean = defaultdict(float)
        self.feature_val_count = defaultdict(int)

        # global stats
        self.feature_stats = defaultdict(AttributionStats)
        self.feature_history = defaultdict(lambda: deque(maxlen=self.window_size))

        # global aligned vector histories
        self._vector_history = deque(maxlen=self.window_size)
        self._abs_attr_vector_history = deque(maxlen=self.window_size)

        # global group/node views
        self.group_stats = defaultdict(lambda: defaultdict(AttributionStats))
        self.node_stats = defaultdict(lambda: defaultdict(AttributionStats))

        # per-attribute / per-intersection explainability views
        self.attribute_stats = defaultdict(lambda: defaultdict(AttributionStats))      # attr -> feature -> stats
        self.attribute_group_stats = defaultdict(lambda: defaultdict(lambda: defaultdict(AttributionStats)))  # attr -> group -> feature -> stats
        self.intersection_stats = defaultdict(lambda: defaultdict(AttributionStats))   # inter_name -> feature -> stats
        self.intersection_group_stats = defaultdict(lambda: defaultdict(lambda: defaultdict(AttributionStats)))  # inter_name -> group -> feature -> stats

        # aligned drift histories per attribute / intersection
        self.attribute_vector_history = defaultdict(lambda: deque(maxlen=self.window_size))
        self.attribute_abs_vector_history = defaultdict(lambda: deque(maxlen=self.window_size))
        self.intersection_vector_history = defaultdict(lambda: deque(maxlen=self.window_size))
        self.intersection_abs_vector_history = defaultdict(lambda: deque(maxlen=self.window_size))

        # scope context
        self.scope_state: Dict[str, Any] = {}
        self.fairness_scope_snapshot: Dict[str, Any] = {}
        self.active_sensitive_dict: Dict[str, Any] = {}

        self._t = 0

    # ============================================================
    # Scope helpers
    # ============================================================

    def update_scope_context(
        self,
        scope_state: Optional[Dict[str, Any]] = None,
        fairness_scope_snapshot: Optional[Dict[str, Any]] = None,
        active_sensitive_dict: Optional[Dict[str, Any]] = None,
    ) -> None:
        if isinstance(scope_state, dict):
            self.scope_state = dict(scope_state)
        if isinstance(fairness_scope_snapshot, dict):
            self.fairness_scope_snapshot = dict(fairness_scope_snapshot)
        if isinstance(active_sensitive_dict, dict):
            self.active_sensitive_dict = dict(active_sensitive_dict)

    def _active_scope_attributes(self, fallback_sensitive: Optional[Dict[str, str]] = None) -> List[str]:
        attrs = []
        if isinstance(self.scope_state, dict):
            attrs = self.scope_state.get("active_attributes", [])
        if not isinstance(attrs, list) or not attrs:
            if isinstance(fallback_sensitive, dict) and fallback_sensitive:
                attrs = list(sorted(fallback_sensitive.keys()))
        return [str(a) for a in attrs]

    def _active_scope_intersections(self, fallback_sensitive: Optional[Dict[str, str]] = None) -> List[Tuple[str, ...]]:
        raw = []
        if isinstance(self.scope_state, dict):
            raw = self.scope_state.get("active_intersections", [])
        out = _normalize_intersection_specs(raw)
        if out:
            return out[: self.max_intersections]

        if isinstance(fallback_sensitive, dict):
            keys = sorted(fallback_sensitive.keys())
            combos = []
            if len(keys) >= 2:
                for c in combinations(keys, 2):
                    combos.append(tuple(c))
                    if len(combos) >= self.max_intersections:
                        break
            return combos
        return []

    def _primary_scope_attribute(self, fallback_sensitive: Optional[Dict[str, str]] = None) -> Optional[str]:
        if isinstance(self.scope_state, dict):
            p = self.scope_state.get("primary_attribute")
            if p is not None:
                return str(p)
        attrs = self._active_scope_attributes(fallback_sensitive=fallback_sensitive)
        return attrs[0] if attrs else None

    def _active_scope_payload(self) -> Dict[str, Any]:
        attrs = self._active_scope_attributes()
        inters = self._active_scope_intersections()
        primary = self._primary_scope_attribute()

        attribute_metrics = self._attribute_metrics_payload()
        active_attr_metrics = {a: attribute_metrics.get(a, {}) for a in attrs}
        attr_debt = {
            a: float(attribute_metrics.get(a, {}).get("explainability_debt", 0.0))
            for a in attrs
        }

        return {
            "mode": self.scope_state.get("mode", "unknown") if isinstance(self.scope_state, dict) else "unknown",
            "primary_attribute": primary,
            "active_attributes": attrs,
            "active_intersections": inters,
            "active_attribute_metrics": active_attr_metrics,
            "active_attribute_debt": attr_debt,
            "scope_snapshot_profiles": self.fairness_scope_snapshot.get("profiles", {})
            if isinstance(self.fairness_scope_snapshot, dict)
            else {},
        }

    # ============================================================
    # Prediction helper
    # ============================================================

    def _predict_score(self, model: Any, x: Dict[str, Any], y_true: Optional[Any] = None) -> Tuple[float, Optional[Any]]:
        try:
            proba = model.predict_proba_one(x)
        except Exception:
            proba = None

        if isinstance(proba, dict) and proba:
            if 1 in proba or True in proba:
                s = _safe_float(proba.get(1, proba.get(True, 0.0)), 0.0)
                s = float(np.clip(s, 0.0, 1.0))
                yhat = max(proba, key=proba.get)
                return s, yhat

            if y_true is not None and y_true in proba:
                s = _safe_float(proba.get(y_true, 0.0), 0.0)
                s = float(np.clip(s, 0.0, 1.0))
                yhat = max(proba, key=proba.get)
                return s, yhat

            yhat = max(proba, key=proba.get)
            s = _safe_float(proba.get(yhat, 0.0), 0.0)
            s = float(np.clip(s, 0.0, 1.0))
            return s, yhat

        try:
            yhat = model.predict_one(x)
        except Exception:
            yhat = None

        if y_true is not None and yhat is not None:
            return (1.0 if yhat == y_true else 0.0), yhat

        return 0.0, yhat

    # ============================================================
    # Vocabulary + imputation
    # ============================================================

    def _update_vocab_and_means(self, x: Dict[str, Any]) -> None:
        for f, v in x.items():
            if f not in self._known_set:
                if len(self.known_features) >= self.max_features:
                    continue
                self._known_set.add(f)
                self.known_features.append(f)

            val = _safe_float(v, default=None)  # type: ignore
            if val is None:
                continue
            c = self.feature_val_count[f]
            self.feature_val_count[f] = c + 1
            self.feature_val_mean[f] = self.feature_val_mean[f] + (val - self.feature_val_mean[f]) / (c + 1)

    def _impute(self, feature: str) -> float:
        if self.impute_strategy == "zero":
            return 0.0
        return float(self.feature_val_mean.get(feature, 0.0))

    # ============================================================
    # MAIN UPDATE
    # ============================================================

    def update(
        self,
        model: Any,
        x: Dict[str, Any],
        y_true: Optional[Any] = None,
        sensitive: Optional[Any] = None,
        node_id: Optional[Any] = None,
        scope_state: Optional[Dict[str, Any]] = None,
        fairness_scope_snapshot: Optional[Dict[str, Any]] = None,
        active_sensitive_dict: Optional[Dict[str, Any]] = None,
    ) -> Dict[str, Any]:
        self._t += 1

        if scope_state is not None or fairness_scope_snapshot is not None or active_sensitive_dict is not None:
            self.update_scope_context(
                scope_state=scope_state,
                fairness_scope_snapshot=fairness_scope_snapshot,
                active_sensitive_dict=active_sensitive_dict,
            )

        do_update = (self._t % self.update_every == 0)

        try:
            self._update_vocab_and_means(x)
        except Exception:
            pass

        sens_norm = normalize_sensitive(sensitive) if sensitive is not None else (("attr", "unknown"),)
        sens_dict = {k: v for k, v in sens_norm}

        if not do_update:
            return {
                "local_importance": {},
                "global_drift": float(self._compute_global_drift()),
                "distribution_drift": float(self._compute_distribution_drift()),
                "top_features": _stable_topk(((k, v.ema) for k, v in self.feature_stats.items()), self.top_k),
                "attribute_metrics": self._attribute_metrics_payload(),
                "intersection_metrics": self._intersection_metrics_payload(),
                "active_scope": self._active_scope_payload(),
            }

        base_score, _ = self._predict_score(model, x, y_true=y_true)

        err_w = 1.0
        if y_true is not None:
            yt = 1.0 if (y_true == 1 or y_true is True) else 0.0
            err_w = abs(yt - float(base_score))
        err_w = float(np.clip(err_w, 0.0, 1.0))

        feats = list(self.known_features)
        if not feats:
            return {
                "local_importance": {},
                "global_drift": 0.0,
                "distribution_drift": 0.0,
                "top_features": [],
                "attribute_metrics": {},
                "intersection_metrics": {},
                "active_scope": self._active_scope_payload(),
            }

        if len(feats) > self.check_features_per_update:
            if self.sample_features:
                feats_to_check = self.rng.sample(feats, self.check_features_per_update)
            else:
                feats_to_check = feats[: self.check_features_per_update]
        else:
            feats_to_check = feats

        active_attrs = self._active_scope_attributes(fallback_sensitive=sens_dict)
        active_inters = self._active_scope_intersections(fallback_sensitive=sens_dict)

        local_importance: Dict[str, float] = {}

        for f in feats_to_check:
            if f not in x:
                continue
            try:
                x_mod = dict(x)
                x_mod[f] = self._impute(f)

                mod_score, _ = self._predict_score(model, x_mod, y_true=y_true)
                contrib = (float(base_score) - float(mod_score)) * err_w
                contrib = _safe_float(contrib, 0.0)
                local_importance[f] = contrib

                # global
                self.feature_history[f].append(contrib)
                hist_mean = float(np.mean(self.feature_history[f])) if self.feature_history[f] else contrib

                st = self.feature_stats[f]
                st.ema = _ema_update(st.ema, hist_mean, self.ema_alpha)
                st.count += 1

                # full-group / node
                if sensitive is not None:
                    g = _sensitive_to_label(sens_norm)
                    gst = self.group_stats[g][f]
                    gst.ema = _ema_update(gst.ema, contrib, self.ema_alpha)
                    gst.count += 1

                if node_id is not None:
                    nid = str(node_id)
                    nst = self.node_stats[nid][f]
                    nst.ema = _ema_update(nst.ema, contrib, self.ema_alpha)
                    nst.count += 1

                # per-attribute
                for attr in active_attrs:
                    ast = self.attribute_stats[attr][f]
                    ast.ema = _ema_update(ast.ema, contrib, self.ema_alpha)
                    ast.count += 1

                    attr_group = str(sens_dict.get(attr, "unknown"))
                    agst = self.attribute_group_stats[attr][attr_group][f]
                    agst.ema = _ema_update(agst.ema, contrib, self.ema_alpha)
                    agst.count += 1

                # per-intersection
                for inter in active_inters:
                    inter_name = _intersection_name(inter)
                    ist = self.intersection_stats[inter_name][f]
                    ist.ema = _ema_update(ist.ema, contrib, self.ema_alpha)
                    ist.count += 1

                    inter_group = _sensitive_to_label(_pair_list_from_sensitive_dict(sens_dict, inter))
                    igst = self.intersection_group_stats[inter_name][inter_group][f]
                    igst.ema = _ema_update(igst.ema, contrib, self.ema_alpha)
                    igst.count += 1

            except Exception:
                continue

        # global aligned histories
        top_global = _stable_topk(((k, v.ema) for k, v in self.feature_stats.items()), self.top_k)
        aligned_feats = [k for k, _ in top_global]
        if aligned_feats:
            vec_map = {f: float(self.feature_stats[f].ema) for f in aligned_feats}
            abs_map = {f: float(abs(self.feature_stats[f].ema)) for f in aligned_feats}
            self._vector_history.append(vec_map)
            self._abs_attr_vector_history.append(abs_map)

        # per-attribute aligned histories
        for attr in active_attrs:
            top_attr = _stable_topk(((k, v.ema) for k, v in self.attribute_stats[attr].items()), self.top_k)
            attr_feats = [k for k, _ in top_attr]
            if attr_feats:
                vec_map = {f: float(self.attribute_stats[attr][f].ema) for f in attr_feats}
                abs_map = {f: float(abs(self.attribute_stats[attr][f].ema)) for f in attr_feats}
                self.attribute_vector_history[attr].append(vec_map)
                self.attribute_abs_vector_history[attr].append(abs_map)

        # per-intersection aligned histories
        for inter in active_inters:
            inter_name = _intersection_name(inter)
            top_inter = _stable_topk(((k, v.ema) for k, v in self.intersection_stats[inter_name].items()), self.top_k)
            inter_feats = [k for k, _ in top_inter]
            if inter_feats:
                vec_map = {f: float(self.intersection_stats[inter_name][f].ema) for f in inter_feats}
                abs_map = {f: float(abs(self.intersection_stats[inter_name][f].ema)) for f in inter_feats}
                self.intersection_vector_history[inter_name].append(vec_map)
                self.intersection_abs_vector_history[inter_name].append(abs_map)

        return {
            "local_importance": local_importance,
            "global_drift": float(self._compute_global_drift()),
            "distribution_drift": float(self._compute_distribution_drift()),
            "top_features": top_global,
            "attribute_metrics": self._attribute_metrics_payload(),
            "intersection_metrics": self._intersection_metrics_payload(),
            "active_scope": self._active_scope_payload(),
        }

    # ============================================================
    # Drift computations
    # ============================================================

    def _compute_vector_drift(self, history: deque) -> float:
        if len(history) < self.drift_min_history:
            return 0.0
        hist = list(history)
        if len(hist) < 2 * self.drift_chunk_min:
            return 0.0
        mid = len(hist) // 2
        old_maps = hist[:mid]
        new_maps = hist[mid:]

        feat_set = set()
        for m in old_maps:
            feat_set.update(m.keys())
        for m in new_maps:
            feat_set.update(m.keys())
        if not feat_set:
            return 0.0
        feats = sorted(feat_set)

        old_arr = np.array([[float(m.get(f, 0.0)) for f in feats] for m in old_maps], dtype=float)
        new_arr = np.array([[float(m.get(f, 0.0)) for f in feats] for m in new_maps], dtype=float)
        old = np.mean(old_arr, axis=0)
        new = np.mean(new_arr, axis=0)
        return _cosine_drift(old, new)

    def _compute_abs_distribution_drift(self, history: deque) -> float:
        if len(history) < self.drift_min_history:
            return 0.0
        hist = list(history)
        if len(hist) < 2 * self.drift_chunk_min:
            return 0.0
        mid = len(hist) // 2
        old_maps = hist[:mid]
        new_maps = hist[mid:]

        feat_set = set()
        for m in old_maps:
            feat_set.update(m.keys())
        for m in new_maps:
            feat_set.update(m.keys())
        if not feat_set:
            return 0.0
        feats = sorted(feat_set)

        p = np.mean(np.array([[float(m.get(f, 0.0)) for f in feats] for m in old_maps], dtype=float), axis=0)
        q = np.mean(np.array([[float(m.get(f, 0.0)) for f in feats] for m in new_maps], dtype=float), axis=0)

        p = np.maximum(p, 1e-12)
        q = np.maximum(q, 1e-12)
        p = p / np.sum(p)
        q = q / np.sum(q)

        kl_pq = float(np.sum(p * np.log(p / q)))
        kl_qp = float(np.sum(q * np.log(q / p)))
        val = 0.5 * (kl_pq + kl_qp)
        return float(np.tanh(val))

    def _compute_global_drift(self) -> float:
        return self._compute_vector_drift(self._vector_history)

    def _compute_distribution_drift(self) -> float:
        return self._compute_abs_distribution_drift(self._abs_attr_vector_history)

    def _compute_attribute_drift(self, attr: str) -> float:
        return self._compute_vector_drift(self.attribute_vector_history.get(attr, deque()))

    def _compute_attribute_distribution_drift(self, attr: str) -> float:
        return self._compute_abs_distribution_drift(self.attribute_abs_vector_history.get(attr, deque()))

    def _compute_intersection_drift(self, inter_name: str) -> float:
        return self._compute_vector_drift(self.intersection_vector_history.get(inter_name, deque()))

    def _compute_intersection_distribution_drift(self, inter_name: str) -> float:
        return self._compute_abs_distribution_drift(self.intersection_abs_vector_history.get(inter_name, deque()))

    # ============================================================
    # Node & group inspection APIs
    # ============================================================

    def get_node_importance(self, node_id: Any, top_k: Optional[int] = None) -> List[Tuple[str, float]]:
        nid = str(node_id)
        if nid not in self.node_stats:
            return []
        k = self.top_k if top_k is None else int(max(1, top_k))
        return _stable_topk(((f, st.ema) for f, st in self.node_stats[nid].items()), k)

    def get_group_importance(self, sensitive_group: Any, top_k: Optional[int] = None) -> List[Tuple[str, float]]:
        g = _normalize_group(sensitive_group)
        if g not in self.group_stats:
            return []
        k = self.top_k if top_k is None else int(max(1, top_k))
        return _stable_topk(((f, st.ema) for f, st in self.group_stats[g].items()), k)

    def get_attribute_importance(self, attr: str, top_k: Optional[int] = None) -> List[Tuple[str, float]]:
        if attr not in self.attribute_stats:
            return []
        k = self.top_k if top_k is None else int(max(1, top_k))
        return _stable_topk(((f, st.ema) for f, st in self.attribute_stats[attr].items()), k)

    def get_attribute_group_importance(self, attr: str, group: Any, top_k: Optional[int] = None) -> List[Tuple[str, float]]:
        g = _normalize_group(group)
        if attr not in self.attribute_group_stats or g not in self.attribute_group_stats[attr]:
            return []
        k = self.top_k if top_k is None else int(max(1, top_k))
        return _stable_topk(((f, st.ema) for f, st in self.attribute_group_stats[attr][g].items()), k)

    def get_intersection_importance(self, inter_name: str, top_k: Optional[int] = None) -> List[Tuple[str, float]]:
        if inter_name not in self.intersection_stats:
            return []
        k = self.top_k if top_k is None else int(max(1, top_k))
        return _stable_topk(((f, st.ema) for f, st in self.intersection_stats[inter_name].items()), k)

    # ============================================================
    # Scope payloads
    # ============================================================

    def _attribute_metrics_payload(self) -> Dict[str, Dict[str, Any]]:
        attrs = set(self.attribute_stats.keys())
        active_attrs = set(self._active_scope_attributes())
        primary_attr = self._primary_scope_attribute()

        out: Dict[str, Dict[str, Any]] = {}
        for attr in sorted(attrs):
            top_features = self.get_attribute_importance(attr, top_k=self.top_k)

            # group separation proxy: range of absolute feature attribution means across groups
            group_maps = self.attribute_group_stats.get(attr, {})
            group_separation = 0.0
            if group_maps:
                feat_set = set()
                for gm in group_maps.values():
                    feat_set.update(gm.keys())
                for feat in feat_set:
                    vals = [abs(float(gm.get(feat, AttributionStats()).ema)) for gm in group_maps.values()]
                    if len(vals) >= 2:
                        group_separation = max(group_separation, float(max(vals) - min(vals)))

            global_vals = np.array([abs(float(v)) for _, v in top_features], dtype=float) if top_features else np.array([], dtype=float)
            explainability_strength = float(np.mean(global_vals)) if global_vals.size else 0.0
            drift = float(self._compute_attribute_drift(attr))
            dist_drift = float(self._compute_attribute_distribution_drift(attr))

            out[attr] = {
                "top_features": top_features,
                "drift": drift,
                "distribution_drift": dist_drift,
                "group_separation": float(group_separation),
                "explainability_strength": float(explainability_strength),
                "explainability_debt": float(drift + dist_drift),
                "groups_tracked": int(len(group_maps)),
                "is_active": bool(attr in active_attrs),
                "is_primary": bool(attr == primary_attr),
                "scope_profile": self.fairness_scope_snapshot.get("profiles", {}).get(attr, {})
                if isinstance(self.fairness_scope_snapshot, dict)
                else {},
            }

        return out

    def _intersection_metrics_payload(self) -> Dict[str, Dict[str, Any]]:
        inters = set(self.intersection_stats.keys())
        active_inter_names = {_intersection_name(x) for x in self._active_scope_intersections()}

        out: Dict[str, Dict[str, Any]] = {}
        for inter_name in sorted(inters):
            top_features = self.get_intersection_importance(inter_name, top_k=self.top_k)
            group_maps = self.intersection_group_stats.get(inter_name, {})

            group_separation = 0.0
            if group_maps:
                feat_set = set()
                for gm in group_maps.values():
                    feat_set.update(gm.keys())
                for feat in feat_set:
                    vals = [abs(float(gm.get(feat, AttributionStats()).ema)) for gm in group_maps.values()]
                    if len(vals) >= 2:
                        group_separation = max(group_separation, float(max(vals) - min(vals)))

            global_vals = np.array([abs(float(v)) for _, v in top_features], dtype=float) if top_features else np.array([], dtype=float)
            explainability_strength = float(np.mean(global_vals)) if global_vals.size else 0.0

            out[inter_name] = {
                "top_features": top_features,
                "drift": float(self._compute_intersection_drift(inter_name)),
                "distribution_drift": float(self._compute_intersection_distribution_drift(inter_name)),
                "group_separation": float(group_separation),
                "explainability_strength": float(explainability_strength),
                "groups_tracked": int(len(group_maps)),
                "is_active": bool(inter_name in active_inter_names),
            }

        return out

    def scope_snapshot(self) -> Dict[str, Any]:
        return {
            "scope_state": dict(self.scope_state) if isinstance(self.scope_state, dict) else {},
            "fairness_scope_snapshot": dict(self.fairness_scope_snapshot)
            if isinstance(self.fairness_scope_snapshot, dict)
            else {},
            "active_scope": self._active_scope_payload(),
            "attribute_metrics": self._attribute_metrics_payload(),
            "intersection_metrics": self._intersection_metrics_payload(),
        }

    def snapshot(self) -> Dict[str, Any]:
        attribute_metrics = self._attribute_metrics_payload()
        intersection_metrics = self._intersection_metrics_payload()
        active_scope = self._active_scope_payload()

        return {
            "timestep": int(self._t),
            "global_drift": float(self._compute_global_drift()),
            "distribution_drift": float(self._compute_distribution_drift()),
            "top_features": _stable_topk(((k, v.ema) for k, v in self.feature_stats.items()), self.top_k),
            "attribute_metrics": attribute_metrics,
            "intersection_metrics": intersection_metrics,
            "attribute_debt": {
                k: float(v.get("explainability_debt", 0.0))
                for k, v in attribute_metrics.items()
            },
            "active_scope": active_scope,
            "scope_state": dict(self.scope_state) if isinstance(self.scope_state, dict) else {},
            "fairness_scope_snapshot": dict(self.fairness_scope_snapshot)
            if isinstance(self.fairness_scope_snapshot, dict)
            else {},
            "scope_snapshot": self.scope_snapshot(),
            "known_feature_count": int(len(self.known_features)),
        }

    def reset(self) -> None:
        self.known_features.clear()
        self._known_set.clear()

        self.feature_val_mean.clear()
        self.feature_val_count.clear()

        self.feature_stats.clear()
        self.feature_history.clear()

        self.group_stats.clear()
        self.node_stats.clear()

        self.attribute_stats.clear()
        self.attribute_group_stats.clear()
        self.intersection_stats.clear()
        self.intersection_group_stats.clear()

        self._vector_history.clear()
        self._abs_attr_vector_history.clear()
        self.attribute_vector_history.clear()
        self.attribute_abs_vector_history.clear()
        self.intersection_vector_history.clear()
        self.intersection_abs_vector_history.clear()

        self.scope_state = {}
        self.fairness_scope_snapshot = {}
        self.active_sensitive_dict = {}

        self._t = 0