From aad33612fcda8dbde6bd07f5726933565dad8e4a Mon Sep 17 00:00:00 2001
From: christopher-earl <cearl@umass.edu>
Date: Fri, 24 Jul 2026 09:34:19 +0900
Subject: [PATCH 01/14] Reduce CPU->GPU transfer every time step with learning.

---
 bindsnet/learning/MCC_learning.py | 29 ++++++++++++++++++++++-------
 1 file changed, 22 insertions(+), 7 deletions(-)

diff --git a/bindsnet/learning/MCC_learning.py b/bindsnet/learning/MCC_learning.py
index 33ebacd0a..3fe2014c9 100644
--- a/bindsnet/learning/MCC_learning.py
+++ b/bindsnet/learning/MCC_learning.py
@@ -503,13 +503,28 @@ def _connection_update(self, **kwargs) -> None:
         source_s = self.source.s.view(batch_size, -1).float()
         target_s = self.target.s.view(batch_size, -1).float()
 
-        # Parse keyword arguments.
+        # Reward from current time step
         reward = kwargs["reward"]
-        a_plus = torch.tensor(
-            kwargs.get("a_plus", 1.0), device=self.feature_value.device
+
+        # Build learning-rate and decay tensors
+        if not hasattr(self, "_a_plus_default"):
+            dev = self.feature_value.device
+            self._a_plus_default = torch.tensor(1.0, device=dev)
+            self._a_minus_default = torch.tensor(-1.0, device=dev)
+            dt = float(self.connection.dt)
+            self._decay_plus = torch.exp(-dt / self.tc_plus.to(dev))
+            self._decay_minus = torch.exp(-dt / self.tc_minus.to(dev))
+        a_plus = kwargs.get("a_plus", None)
+        a_plus = (
+            self._a_plus_default
+            if a_plus is None
+            else torch.as_tensor(a_plus, device=self.feature_value.device)
         )
-        a_minus = torch.tensor(
-            kwargs.get("a_minus", -1.0), device=self.feature_value.device
+        a_minus = kwargs.get("a_minus", None)
+        a_minus = (
+            self._a_minus_default
+            if a_minus is None
+            else torch.as_tensor(a_minus, device=self.feature_value.device)
         )
 
         # Compute weight update based on the eligibility value of the past timestep.
@@ -535,9 +550,9 @@ def _connection_update(self, **kwargs) -> None:
             self.feature_value += update
 
         # Update P^+ and P^- values.
-        self.p_plus *= torch.exp(-self.connection.dt / self.tc_plus)
+        self.p_plus *= self._decay_plus
         self.p_plus += a_plus * source_s
-        self.p_minus *= torch.exp(-self.connection.dt / self.tc_minus)
+        self.p_minus *= self._decay_minus
         self.p_minus += a_minus * target_s
 
         # Calculate point eligibility value.

From 1b6e4a2e56d46106b7f4b18c32adf6fd52c8b435 Mon Sep 17 00:00:00 2001
From: christopher-earl <cearl@umass.edu>
Date: Fri, 24 Jul 2026 10:00:06 +0900
Subject: [PATCH 02/14] Foldable pipelines

---
 bindsnet/network/topology.py | 62 ++++++++++++++++++++++++++++++++++++
 1 file changed, 62 insertions(+)

diff --git a/bindsnet/network/topology.py b/bindsnet/network/topology.py
index 10df45824..42d687030 100644
--- a/bindsnet/network/topology.py
+++ b/bindsnet/network/topology.py
@@ -413,6 +413,7 @@ def __init__(
         pipeline: list = [],
         manual_update: bool = False,
         traces: bool = False,
+        compute_dtype: Optional[torch.dtype] = None,
         **kwargs,
     ) -> None:
         # language=rst
@@ -426,6 +427,8 @@ def __init__(
         :param manual_update: Set to :code:`True` to disable automatic updates (applying learning rules) to connection features.
             False by default, updates called after each time step
         :param traces: Set to :code:`True` to record history of connection activity (for monitors)
+        :param compute_dtype: Optional low-precision dtype (e.g. ``torch.bfloat16``)
+            for the matmul fast path only; ``None`` (default) keeps full ``float32``.
         """
 
         super().__init__(source, target, device, pipeline, **kwargs)
@@ -434,6 +437,49 @@ def __init__(
         if self.traces:
             self.activity = None
 
+        self._w_eff = None          # 'Folded' weight matrix for faster computing
+        self._has_learning = None
+        self.compute_dtype = compute_dtype
+
+    def _pipeline_has_learning(self) -> bool:
+        # language=rst
+        """Whether any pipeline feature carries a real (non-``NoOp``) learning
+        rule, i.e. whether the folded-weight cache must be invalidated on update."""
+        if self._has_learning is None:
+            from ..learning.MCC_learning import NoOp
+
+            self._has_learning = any(
+                not isinstance(getattr(f, "learning_rule", None), NoOp)
+                for f in self.pipeline
+            )
+        return self._has_learning
+
+    def _folded_weight(self) -> Optional[torch.Tensor]:
+        # language=rst
+        """
+        Return the product of the pipeline's foldable feature values. If the
+        pipeline is empty or contains a non-foldable feature, returns none.
+        """
+        if not self.pipeline:
+            return None
+        if self._w_eff is not None:
+            return self._w_eff
+        w_eff = None
+        for f in self.pipeline:
+            v = f.matmul_fold_value()
+            if v is None:
+                return None  # a non-foldable feature -> use the generic path
+            w_eff = v if w_eff is None else w_eff * v
+        # A product of only boolean masks would be non-float; force the matmul
+        # dtype (float32 by default, or the opt-in low-precision compute_dtype).
+        if self.compute_dtype is not None:
+            w_eff = w_eff.to(self.compute_dtype)
+        elif not torch.is_floating_point(w_eff):
+            w_eff = w_eff.float()
+        if not self.manual_update:
+            self._w_eff = w_eff
+        return w_eff
+
     def compute(self, s: torch.Tensor) -> torch.Tensor:
         # language=rst
         """
@@ -444,6 +490,18 @@ def compute(self, s: torch.Tensor) -> torch.Tensor:
                  decaying spike activation).
         """
 
+        # Fast path: when the whole pipeline folds to a static elementwise-multiply
+        w_eff = self._folded_weight()
+        if w_eff is not None:
+            out_signal = s.view(s.size(0), self.source.n).to(w_eff.dtype) @ w_eff
+            if out_signal.dtype != torch.float32:
+                out_signal = out_signal.float()
+            if self.traces:
+                self.activity = out_signal
+            if out_signal.size() != torch.Size([s.size(0)] + self.target.shape):
+                return out_signal.view(s.size(0), *self.target.shape)
+            return out_signal
+
         # Change to numeric type (torch doesn't like booleans for matrix ops)
         # Note: .float() is an expensive operation. Use as minimally as possible!
         # if s.dtype != torch.float32:
@@ -516,6 +574,8 @@ def update(self, **kwargs) -> None:
             # Pipeline learning
             for f in self.pipeline:
                 f.update(**kwargs)
+            if self._pipeline_has_learning():
+                self._w_eff = None  # weights changed -> rebuild fold next compute
 
     def normalize(self) -> None:
         # language=rst
@@ -525,6 +585,7 @@ def normalize(self) -> None:
         # Normalize pipeline features
         for f in self.pipeline:
             f.normalize()
+        self._w_eff = None  # normalization may change weights -> invalidate fold
 
     def reset_state_variables(self) -> None:
         # language=rst
@@ -535,6 +596,7 @@ def reset_state_variables(self) -> None:
 
         for f in self.pipeline:
             f.reset_state_variables()
+        self._w_eff = None  # rebuild the fold cache at the next sample
 
 
 class Conv1dConnection(AbstractConnection):

From 2428476a5d8d4aea4561bd70cabfad307e04c68f Mon Sep 17 00:00:00 2001
From: christopher-earl <cearl@umass.edu>
Date: Fri, 24 Jul 2026 10:01:02 +0900
Subject: [PATCH 03/14] More files for foldable pipelines

---
 bindsnet/network/topology_features.py | 29 +++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/bindsnet/network/topology_features.py b/bindsnet/network/topology_features.py
index 43ccc1389..c39288020 100644
--- a/bindsnet/network/topology_features.py
+++ b/bindsnet/network/topology_features.py
@@ -170,6 +170,21 @@ def compute(self, conn_spikes) -> Union[torch.Tensor, float, int]:
         """
         pass
 
+    def matmul_fold_value(self) -> Optional[torch.Tensor]:
+        # language=rst
+        """
+        Fast-path hook for :class:`MulticompartmentConnection`.
+
+        If this feature is a *pure elementwise multiply* by a static
+        ``[source.n, target.n]`` tensor -- i.e. ``compute(x) == x * V`` with no
+        per-call side effects on ``V`` -- return that ``V`` so the connection can
+        fold the whole pipeline into one weight matrix and evaluate as a matmul
+        (``out = s @ W_eff``) instead of materialising ``[B, source.n,
+        target.n]``. Returning ``None`` (the safe default) forces the generic
+        pipeline path.
+        """
+        return None
+
     def prime_feature(self, connection, device, **kwargs) -> None:
         # language=rst
         """
@@ -507,6 +522,12 @@ def __init__(
     def compute(self, conn_spikes) -> torch.Tensor:
         return conn_spikes * self.value
 
+    def matmul_fold_value(self) -> Optional[torch.Tensor]:
+        # A boolean mask is a pure elementwise multiply (True->1, False->0).
+        if getattr(self, "sparse", False):
+            return None
+        return self.value
+
     def reset_state_variables(self) -> None:
         pass
 
@@ -644,6 +665,14 @@ def compute(self, conn_spikes) -> Union[torch.Tensor, float, int]:
 
         return return_val
 
+    def matmul_fold_value(self) -> Optional[torch.Tensor]:
+        # Foldable only as a plain multiply: skip when polarity enforcement or
+        # per-time-step normalisation would mutate ``value`` during compute, or
+        # when the value is stored sparse.
+        if self.enforce_polarity or self.norm_frequency == "time step" or self.sparse:
+            return None
+        return self.value
+
     def prime_feature(self, connection, device, **kwargs) -> None:
         #### Initialize value ####
         if self.value is None:

From 677d196e47895a47d60827e5fbc24b2bc8f5c62c Mon Sep 17 00:00:00 2001
From: christopher-earl <cearl@umass.edu>
Date: Sat, 25 Jul 2026 03:38:40 +0900
Subject: [PATCH 04/14] Sparse compute

---
 bindsnet/network/topology.py | 24 +++++++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git a/bindsnet/network/topology.py b/bindsnet/network/topology.py
index 42d687030..07ee8a5b6 100644
--- a/bindsnet/network/topology.py
+++ b/bindsnet/network/topology.py
@@ -414,6 +414,7 @@ def __init__(
         manual_update: bool = False,
         traces: bool = False,
         compute_dtype: Optional[torch.dtype] = None,
+        sparse_compute: bool = False,
         **kwargs,
     ) -> None:
         # language=rst
@@ -429,6 +430,8 @@ def __init__(
         :param traces: Set to :code:`True` to record history of connection activity (for monitors)
         :param compute_dtype: Optional low-precision dtype (e.g. ``torch.bfloat16``)
             for the matmul fast path only; ``None`` (default) keeps full ``float32``.
+        :param sparse_compute: Set to :code:`True` to compute sparse activity (<~20% neurons active)
+        efficiently
         """
 
         super().__init__(source, target, device, pipeline, **kwargs)
@@ -440,6 +443,7 @@ def __init__(
         self._w_eff = None          # 'Folded' weight matrix for faster computing
         self._has_learning = None
         self.compute_dtype = compute_dtype
+        self.sparse_compute = sparse_compute
 
     def _pipeline_has_learning(self) -> bool:
         # language=rst
@@ -480,6 +484,20 @@ def _folded_weight(self) -> Optional[torch.Tensor]:
             self._w_eff = w_eff
         return w_eff
 
+    def _sparse_matmul(self, s_flat: torch.Tensor, w_eff: torch.Tensor) -> torch.Tensor:
+        # language=rst
+        """
+        Activity-sparse form of ``s_flat @ w_eff``. Read only
+        the rows of ``w_eff`` for source neurons that spiked this step. Numerically
+        identical to the dense matmul. More efficient when few source neurons are active.
+        """
+        active = s_flat.any(dim=0).nonzero(as_tuple=False).squeeze(1)
+        if active.numel() == 0:
+            return torch.zeros(
+                s_flat.size(0), self.target.n, device=w_eff.device, dtype=w_eff.dtype
+            )
+        return s_flat[:, active].to(w_eff.dtype) @ w_eff.index_select(0, active)
+
     def compute(self, s: torch.Tensor) -> torch.Tensor:
         # language=rst
         """
@@ -493,7 +511,11 @@ def compute(self, s: torch.Tensor) -> torch.Tensor:
         # Fast path: when the whole pipeline folds to a static elementwise-multiply
         w_eff = self._folded_weight()
         if w_eff is not None:
-            out_signal = s.view(s.size(0), self.source.n).to(w_eff.dtype) @ w_eff
+            s_flat = s.view(s.size(0), self.source.n)
+            if self.sparse_compute:
+                out_signal = self._sparse_matmul(s_flat, w_eff)
+            else:
+                out_signal = s_flat.to(w_eff.dtype) @ w_eff
             if out_signal.dtype != torch.float32:
                 out_signal = out_signal.float()
             if self.traces:

From 884010b09253cfe322c25ebd6ba1d155314fafb3 Mon Sep 17 00:00:00 2001
From: christopher-earl <cearl@umass.edu>
Date: Sun, 26 Jul 2026 16:07:14 +0900
Subject: [PATCH 05/14] Folding pipeline computation

---
 bindsnet/network/topology.py          | 173 +++++++++-----------------
 bindsnet/network/topology_features.py | 113 +++++++++--------
 2 files changed, 115 insertions(+), 171 deletions(-)

diff --git a/bindsnet/network/topology.py b/bindsnet/network/topology.py
index 07ee8a5b6..eaa4485f8 100644
--- a/bindsnet/network/topology.py
+++ b/bindsnet/network/topology.py
@@ -413,7 +413,6 @@ def __init__(
         pipeline: list = [],
         manual_update: bool = False,
         traces: bool = False,
-        compute_dtype: Optional[torch.dtype] = None,
         sparse_compute: bool = False,
         **kwargs,
     ) -> None:
@@ -428,10 +427,8 @@ def __init__(
         :param manual_update: Set to :code:`True` to disable automatic updates (applying learning rules) to connection features.
             False by default, updates called after each time step
         :param traces: Set to :code:`True` to record history of connection activity (for monitors)
-        :param compute_dtype: Optional low-precision dtype (e.g. ``torch.bfloat16``)
-            for the matmul fast path only; ``None`` (default) keeps full ``float32``.
-        :param sparse_compute: Set to :code:`True` to compute sparse activity (<~20% neurons active)
-        efficiently
+        :param sparse_compute: Set to :code:`True` to read only the rows of the effective
+            weight for source neurons that spiked (a win only when few are active).
         """
 
         super().__init__(source, target, device, pipeline, **kwargs)
@@ -440,123 +437,75 @@ def __init__(
         if self.traces:
             self.activity = None
 
-        self._w_eff = None          # 'Folded' weight matrix for faster computing
-        self._has_learning = None
-        self.compute_dtype = compute_dtype
         self.sparse_compute = sparse_compute
 
-    def _pipeline_has_learning(self) -> bool:
-        # language=rst
-        """Whether any pipeline feature carries a real (non-``NoOp``) learning
-        rule, i.e. whether the folded-weight cache must be invalidated on update."""
-        if self._has_learning is None:
-            from ..learning.MCC_learning import NoOp
-
-            self._has_learning = any(
-                not isinstance(getattr(f, "learning_rule", None), NoOp)
-                for f in self.pipeline
-            )
-        return self._has_learning
-
-    def _folded_weight(self) -> Optional[torch.Tensor]:
-        # language=rst
-        """
-        Return the product of the pipeline's foldable feature values. If the
-        pipeline is empty or contains a non-foldable feature, returns none.
-        """
-        if not self.pipeline:
-            return None
-        if self._w_eff is not None:
-            return self._w_eff
-        w_eff = None
-        for f in self.pipeline:
-            v = f.matmul_fold_value()
-            if v is None:
-                return None  # a non-foldable feature -> use the generic path
-            w_eff = v if w_eff is None else w_eff * v
-        # A product of only boolean masks would be non-float; force the matmul
-        # dtype (float32 by default, or the opt-in low-precision compute_dtype).
-        if self.compute_dtype is not None:
-            w_eff = w_eff.to(self.compute_dtype)
-        elif not torch.is_floating_point(w_eff):
-            w_eff = w_eff.float()
-        if not self.manual_update:
-            self._w_eff = w_eff
-        return w_eff
-
-    def _sparse_matmul(self, s_flat: torch.Tensor, w_eff: torch.Tensor) -> torch.Tensor:
-        # language=rst
-        """
-        Activity-sparse form of ``s_flat @ w_eff``. Read only
-        the rows of ``w_eff`` for source neurons that spiked this step. Numerically
-        identical to the dense matmul. More efficient when few source neurons are active.
-        """
-        active = s_flat.any(dim=0).nonzero(as_tuple=False).squeeze(1)
-        if active.numel() == 0:
-            return torch.zeros(
-                s_flat.size(0), self.target.n, device=w_eff.device, dtype=w_eff.dtype
-            )
-        return s_flat[:, active].to(w_eff.dtype) @ w_eff.index_select(0, active)
-
     def compute(self, s: torch.Tensor) -> torch.Tensor:
         # language=rst
         """
-        Compute pre-activations given spikes using connection weights.
+        Direct incoming spikes through the connection's feature pipeline.
 
-        :param s: Incoming spikes.
-        :return: Incoming spikes multiplied by synaptic weights (with or without
-                 decaying spike activation).
-        """
+        Each feature's ``compute`` returns its ``[source.n, target.n]`` value; how
+        it folds is set by the feature's ``op`` (``"mul"`` default, ``"add"``,
+        ``"sub"``). Folding the recurrence (start ``A = 1``, ``B = 0``):
 
-        # Fast path: when the whole pipeline folds to a static elementwise-multiply
-        w_eff = self._folded_weight()
-        if w_eff is not None:
-            s_flat = s.view(s.size(0), self.source.n)
-            if self.sparse_compute:
-                out_signal = self._sparse_matmul(s_flat, w_eff)
-            else:
-                out_signal = s_flat.to(w_eff.dtype) @ w_eff
-            if out_signal.dtype != torch.float32:
-                out_signal = out_signal.float()
-            if self.traces:
-                self.activity = out_signal
-            if out_signal.size() != torch.Size([s.size(0)] + self.target.shape):
-                return out_signal.view(s.size(0), *self.target.shape)
-            return out_signal
-
-        # Change to numeric type (torch doesn't like booleans for matrix ops)
-        # Note: .float() is an expensive operation. Use as minimally as possible!
-        # if s.dtype != torch.float32:
-        #     s = s.float()
-
-        # Prepare broadcast from incoming spikes to all output neurons
-        # |conn_spikes| = [batch_size, source.n * target.n]
-        conn_spikes = s.view(s.size(0), self.source.n, 1).repeat(1, 1, self.target.n)
-        # TODO: ^ This could probably be optimized
-
-        # Run through pipeline
-        for f in self.pipeline:
-            conn_spikes = f.compute(conn_spikes)
+        * ``mul`` factor ``a``: ``A <- a * A`` and ``B <- a * B``
+        * ``add`` term  ``b``: ``B <- B + b``
+        * ``sub`` term  ``b``: ``B <- B - b``
+
+        ``B`` stays ``None`` (unallocated) unless an additive feature is present,
+        so a purely multiplicative pipeline is exactly the single ``s @ A`` matmul.
 
-        # Sum signals for each of the output/terminal neurons
-        # |out_signal| = [batch_size, target.n]
-        if conn_spikes.size() != torch.Size([s.size(0), self.source.n, self.target.n]):
-            if conn_spikes.is_sparse:
-                conn_spikes = conn_spikes.to_dense()
-            conn_spikes = conn_spikes.view(s.size(0), self.source.n, self.target.n)
+        :param s: Incoming spikes, shape ``[batch, *source.shape]``.
+        :return: Post-synaptic input, shape ``[batch, *target.shape]``.
+        """
+        s = s.view(s.size(0), self.source.n)
 
-        if conn_spikes.is_sparse:
-            out_signal = conn_spikes.to_dense().sum(1)
+        # running product of multiplicative factors, [source.n, target.n]
+        a_eff = None
+        # running additive offset, [source.n, target.n]; None while still zero
+        b_eff = None
+        for f in self.pipeline:
+            factor = f.compute(s)
+            op = getattr(f, "op", "mul")
+            if op == "mul":
+                a_eff = factor if a_eff is None else a_eff * factor
+                if b_eff is not None:
+                    b_eff = b_eff * factor
+            else:  # additive contribution: "add" -> +factor, "sub" -> -factor
+                term = factor if op == "add" else -factor
+                b_eff = term if b_eff is None else b_eff + term
+
+        if a_eff is None:
+            # Degenerate pipeline with no multiplicative feature: every source
+            # neuron contributes with unit weight.
+            a_eff = torch.ones(s.size(1), b_eff.size(-1), device=s.device)
+        if not torch.is_floating_point(a_eff):
+            a_eff = a_eff.float()
+
+        if self.sparse_compute:
+            # Read only the rows of A for source neurons that spiked this step
+            # (numerically identical; faster only when few are active).
+            active = s.any(dim=0).nonzero(as_tuple=False).squeeze(1)
+            if active.numel() == 0:
+                out = torch.zeros(
+                    s.size(0), a_eff.size(-1), device=a_eff.device, dtype=a_eff.dtype
+                )
+            else:
+                out = s[:, active].to(a_eff.dtype) @ a_eff.index_select(0, active)
         else:
-            out_signal = conn_spikes.sum(1)
+            out = s.to(a_eff.dtype) @ a_eff
 
-        if self.traces:
-            self.activity = out_signal
+        # Additive terms apply to every synapse regardless of spikes, so sum over
+        # all source rows (the closed form of the [batch, source.n, target.n]
+        # source-sum).
+        if b_eff is not None:
+            out = out + b_eff.sum(dim=0)
 
-        if out_signal.size() != torch.Size([s.size(0)] + self.target.shape):
-            return out_signal.view(s.size(0), *self.target.shape)
-        else:
-            return out_signal
+        if self.traces:
+            self.activity = out
+        if out.size() != torch.Size([s.size(0)] + self.target.shape):
+            return out.view(s.size(0), *self.target.shape)
+        return out
 
     def compute_window(self, s: torch.Tensor) -> torch.Tensor:
         # language=rst
@@ -596,8 +545,6 @@ def update(self, **kwargs) -> None:
             # Pipeline learning
             for f in self.pipeline:
                 f.update(**kwargs)
-            if self._pipeline_has_learning():
-                self._w_eff = None  # weights changed -> rebuild fold next compute
 
     def normalize(self) -> None:
         # language=rst
@@ -607,7 +554,6 @@ def normalize(self) -> None:
         # Normalize pipeline features
         for f in self.pipeline:
             f.normalize()
-        self._w_eff = None  # normalization may change weights -> invalidate fold
 
     def reset_state_variables(self) -> None:
         # language=rst
@@ -618,7 +564,6 @@ def reset_state_variables(self) -> None:
 
         for f in self.pipeline:
             f.reset_state_variables()
-        self._w_eff = None  # rebuild the fold cache at the next sample
 
 
 class Conv1dConnection(AbstractConnection):
diff --git a/bindsnet/network/topology_features.py b/bindsnet/network/topology_features.py
index c39288020..f6126eba2 100644
--- a/bindsnet/network/topology_features.py
+++ b/bindsnet/network/topology_features.py
@@ -18,6 +18,12 @@ class AbstractFeature(ABC):
     Features to operate on signals traversing a connection.
     """
 
+    # How this feature folds in the connection's affine pipeline (see
+    # :meth:`MulticompartmentConnection.compute`): ``"mul"`` (elementwise multiply,
+    # the default), ``"add"`` or ``"sub"`` (elementwise add/subtract of the value
+    # returned by :meth:`compute`).
+    op = "mul"
+
     @abstractmethod
     def __init__(
         self,
@@ -163,28 +169,16 @@ def reset_state_variables(self) -> None:
         pass
 
     @abstractmethod
-    def compute(self, conn_spikes) -> Union[torch.Tensor, float, int]:
+    def compute(self, s) -> Union[torch.Tensor, float, int]:
         # language=rst
         """
-        Computes the feature being operated on a set of incoming signals.
+        Return this feature's ``[source.n, target.n]`` value, given pre-synaptic
+        spikes ``s`` of shape ``[batch, source.n]``. How the value folds into the
+        connection is set by :attr:`op` -- a multiplicative factor (``"mul"``, the
+        default), or an additive/subtractive offset (``"add"``/``"sub"``).
         """
         pass
 
-    def matmul_fold_value(self) -> Optional[torch.Tensor]:
-        # language=rst
-        """
-        Fast-path hook for :class:`MulticompartmentConnection`.
-
-        If this feature is a *pure elementwise multiply* by a static
-        ``[source.n, target.n]`` tensor -- i.e. ``compute(x) == x * V`` with no
-        per-call side effects on ``V`` -- return that ``V`` so the connection can
-        fold the whole pipeline into one weight matrix and evaluate as a matmul
-        (``out = s @ W_eff``) instead of materialising ``[B, source.n,
-        target.n]``. Returning ``None`` (the safe default) forces the generic
-        pipeline path.
-        """
-        return None
-
     def prime_feature(self, connection, device, **kwargs) -> None:
         # language=rst
         """
@@ -437,11 +431,11 @@ def sparse_bernoulli(self):
         non_zero = values[mask]
         return torch.sparse_coo_tensor(indices, non_zero, self.value.size())
 
-    def compute(self, conn_spikes) -> Union[torch.Tensor, float, int]:
+    def compute(self, s) -> Union[torch.Tensor, float, int]:
+        # Factor: a fresh Bernoulli draw each step (resampled, never cached).
         if self.sparse:
-            return conn_spikes * self.sparse_bernoulli()
-        else:
-            return conn_spikes * torch.bernoulli(self.value)
+            return self.sparse_bernoulli()
+        return torch.bernoulli(self.value)
 
     def reset_state_variables(self) -> None:
         pass
@@ -519,13 +513,7 @@ def __init__(
         self.name = name
         self.value = value
 
-    def compute(self, conn_spikes) -> torch.Tensor:
-        return conn_spikes * self.value
-
-    def matmul_fold_value(self) -> Optional[torch.Tensor]:
-        # A boolean mask is a pure elementwise multiply (True->1, False->0).
-        if getattr(self, "sparse", False):
-            return None
+    def compute(self, s) -> torch.Tensor:
         return self.value
 
     def reset_state_variables(self) -> None:
@@ -581,9 +569,9 @@ def __init__(self) -> None:
     def reset_state_variables(self) -> None:
         pass
 
-    def compute(self, conn_spikes) -> Union[torch.Tensor, float, int]:
-        return conn_spikes.mean() * torch.ones(
-            self.source_n * self.target_n, device=self.device
+    def compute(self, s) -> Union[torch.Tensor, float, int]:
+        return s.float().mean() * torch.ones(
+            self.source_n, self.target_n, device=s.device
         )
 
     def prime_feature(self, connection, device, **kwargs) -> None:
@@ -651,7 +639,7 @@ def __init__(
     def reset_state_variables(self) -> None:
         pass
 
-    def compute(self, conn_spikes) -> Union[torch.Tensor, float, int]:
+    def compute(self, s) -> Union[torch.Tensor, float, int]:
         if self.enforce_polarity:
             pos_mask = ~torch.logical_xor(self.value > 0, self.positive_mask)
             neg_mask = ~torch.logical_xor(self.value < 0, ~self.positive_mask)
@@ -659,19 +647,11 @@ def compute(self, conn_spikes) -> Union[torch.Tensor, float, int]:
             self.value[~pos_mask] = 0.0001
             self.value[~neg_mask] = -0.0001
 
-        return_val = self.value * conn_spikes
+        factor = self.value
         if self.norm_frequency == "time step":
+            factor = factor.clone()
             self.normalize(time_step_norm=True)
-
-        return return_val
-
-    def matmul_fold_value(self) -> Optional[torch.Tensor]:
-        # Foldable only as a plain multiply: skip when polarity enforcement or
-        # per-time-step normalisation would mutate ``value`` during compute, or
-        # when the value is stored sparse.
-        if self.enforce_polarity or self.norm_frequency == "time step" or self.sparse:
-            return None
-        return self.value
+        return factor
 
     def prime_feature(self, connection, device, **kwargs) -> None:
         #### Initialize value ####
@@ -734,11 +714,15 @@ def __init__(
             batch_size=batch_size,
         )
 
+    # Bias is additive: folds as ``B <- B + value`` in the connection's pipeline.
+    op = "add"
+
     def reset_state_variables(self) -> None:
         pass
 
-    def compute(self, conn_spikes) -> Union[torch.Tensor, float, int]:
-        return conn_spikes + self.value
+    def compute(self, s) -> Union[torch.Tensor, float, int]:
+        # Additive offset added to every synapse (independent of the spikes).
+        return self.value
 
     def prime_feature(self, connection, device, **kwargs) -> None:
         #### Initialize value ####
@@ -762,7 +746,7 @@ def __init__(
     ) -> None:
         # language=rst
         """
-        Adds scalars to signals
+        Multiply all signals by a scalar
         :param name: Name of the feature
         :param value: Values to scale signals by
         :param value_dtype: Data type for :code:`value` tensor
@@ -781,8 +765,8 @@ def __init__(
     def reset_state_variables(self) -> None:
         pass
 
-    def compute(self, conn_spikes) -> Union[torch.Tensor, float, int]:
-        return conn_spikes * self.value
+    def compute(self, s) -> Union[torch.Tensor, float, int]:
+        return self.value
 
     def prime_feature(self, connection, device, **kwargs) -> None:
         #### Initialize value ####
@@ -835,11 +819,17 @@ def __init__(
 
         self.degrade_function = degrade_function
 
+    # Degradation is subtractive: folded as ``B <- B - degrade_function(value)``.
+    op = "sub"
+
     def reset_state_variables(self) -> None:
         pass
 
-    def compute(self, conn_spikes) -> Union[torch.Tensor, float, int]:
-        return conn_spikes - self.degrade_function(self.value)
+    def compute(self, s) -> Union[torch.Tensor, float, int]:
+        # Subtractive offset (via degrade_function) applied to every synapse.
+        if self.degrade_function is not None:
+            return self.degrade_function(self.value)
+        return self.value
 
 
 class AdaptationBaseSynapsHistory(AbstractFeature):
@@ -911,7 +901,11 @@ def forward(self, x):
             batch_size=batch_size,
         )
 
-    def compute(self, conn_spikes) -> Union[torch.Tensor, float, int]:
+    def compute(self, s) -> Union[torch.Tensor, float, int]:
+        # This feature needs the per-synapse spikes, so build them from s
+        conn_spikes = s.view(s.size(0), -1, 1).expand(
+            s.size(0), s.size(1), self.value.size(-1)
+        )
 
         # Update the spike buffer
         if self.start_counter == False or conn_spikes.sum() > 0:
@@ -940,7 +934,7 @@ def compute(self, conn_spikes) -> Union[torch.Tensor, float, int]:
             if self.sparse:
                 self.value = self.value.to_sparse()
 
-        return conn_spikes * self.value
+        return self.value
 
     def reset_state_variables(
         self,
@@ -1021,7 +1015,11 @@ def forward(self, x):
             batch_size=batch_size,
         )
 
-    def compute(self, conn_spikes) -> Union[torch.Tensor, float, int]:
+    def compute(self, s) -> Union[torch.Tensor, float, int]:
+        # This feature needs the per-synapse spikes, so build them from s
+        conn_spikes = s.view(s.size(0), -1, 1).expand(
+            s.size(0), s.size(1), self.value.size(-1)
+        )
 
         # Update the spike buffer
         if self.start_counter == False or conn_spikes.sum() > 0:
@@ -1050,7 +1048,7 @@ def compute(self, conn_spikes) -> Union[torch.Tensor, float, int]:
             if self.sparse:
                 self.value = self.value.to_sparse()
 
-        return conn_spikes * self.value
+        return self.value
 
     def reset_state_variables(
         self,
@@ -1089,15 +1087,16 @@ def __init__(
         self.parent = parent_feature
         self.sub_feature = None  # <-- Defined in non-abstract constructor
 
-    def compute(self, _) -> None:
+    def compute(self, s):
         # language=rst
         """
-        Proxy function to catch a pipeline execution from topology.py's :code:`compute` function. Allows :code:`SubFeature`
-        objects to be executed like real features in the pipeline.
+        Proxy to run a parent feature's side-effect (e.g. normalize/update) from
+        inside the pipeline. Returns ``1`` so it is the identity factor in the fold.
         """
 
         # sub_feature should be defined in the non-abstract constructor
         self.sub_feature()
+        return 1
 
 
 class Normalization(AbstractSubFeature):

From 48c98c04bed38b65ffeaa8104c3aa97c1ef4b92e Mon Sep 17 00:00:00 2001
From: christopher-earl <cearl@umass.edu>
Date: Sun, 26 Jul 2026 20:21:56 +0900
Subject: [PATCH 06/14] Fixed indentation error with learning rule class

---
 bindsnet/learning/MCC_learning.py | 137 +++++++++++++++---------------
 1 file changed, 69 insertions(+), 68 deletions(-)

diff --git a/bindsnet/learning/MCC_learning.py b/bindsnet/learning/MCC_learning.py
index 3fe2014c9..660bd538c 100644
--- a/bindsnet/learning/MCC_learning.py
+++ b/bindsnet/learning/MCC_learning.py
@@ -304,89 +304,90 @@ def _connection_update(self, **kwargs) -> None:
     def reset_state_variables(self):
         return
 
-    class Hebbian(MCC_LearningRule):
+
+class Hebbian(MCC_LearningRule):
+    # language=rst
+    """
+    Simple Hebbian learning rule. Pre- and post-synaptic updates are both positive.
+    """
+
+    def __init__(
+        self,
+        connection: AbstractMulticompartmentConnection,
+        feature_value: Union[torch.Tensor, float, int],
+        nu: Optional[Union[float, Sequence[float]]] = None,
+        reduction: Optional[callable] = None,
+        decay: float = 0.0,
+        **kwargs,
+    ) -> None:
         # language=rst
         """
-        Simple Hebbian learning rule. Pre- and post-synaptic updates are both positive.
-        """
+        Constructor for ``Hebbian`` learning rule.
 
-        def __init__(
-            self,
-            connection: AbstractMulticompartmentConnection,
-            feature_value: Union[torch.Tensor, float, int],
-            nu: Optional[Union[float, Sequence[float]]] = None,
-            reduction: Optional[callable] = None,
-            decay: float = 0.0,
+        :param connection: An ``AbstractConnection`` object whose weights the
+            ``Hebbian`` learning rule will modify.
+        :param nu: Single or pair of learning rates for pre- and post-synaptic events.
+        :param reduction: Method for reducing parameter updates along the batch
+            dimension.
+        :param decay: Coefficient controlling rate of decay of the weights each iteration.
+        """
+        super().__init__(
+            connection=connection,
+            feature_value=feature_value,
+            nu=nu,
+            reduction=reduction,
+            decay=decay,
             **kwargs,
-        ) -> None:
-            # language=rst
-            """
-            Constructor for ``Hebbian`` learning rule.
-
-            :param connection: An ``AbstractConnection`` object whose weights the
-                ``Hebbian`` learning rule will modify.
-            :param nu: Single or pair of learning rates for pre- and post-synaptic events.
-            :param reduction: Method for reducing parameter updates along the batch
-                dimension.
-            :param decay: Coefficient controlling rate of decay of the weights each iteration.
-            """
-            super().__init__(
-                connection=connection,
-                feature_value=feature_value,
-                nu=nu,
-                reduction=reduction,
-                decay=decay,
-                **kwargs,
-            )
+        )
 
-            assert (
-                self.source.traces and self.target.traces
-            ), "Both pre- and post-synaptic nodes must record spike traces."
+        assert (
+            self.source.traces and self.target.traces
+        ), "Both pre- and post-synaptic nodes must record spike traces."
 
-            if isinstance(MulticompartmentConnection):
-                self.update = self._connection_update
-                self.feature_value = feature_value
-            # elif isinstance(connection, Conv2dConnection):
-            #     self.update = self._conv2d_connection_update
-            else:
-                raise NotImplementedError(
-                    "This learning rule is not supported for this Connection type."
-                )
+        if isinstance(MulticompartmentConnection):
+            self.update = self._connection_update
+            self.feature_value = feature_value
+        # elif isinstance(connection, Conv2dConnection):
+        #     self.update = self._conv2d_connection_update
+        else:
+            raise NotImplementedError(
+                "This learning rule is not supported for this Connection type."
+            )
 
-        def _connection_update(self, **kwargs) -> None:
-            # language=rst
-            """
-            Hebbian learning rule for ``Connection`` subclass of ``AbstractConnection``
-            class.
-            """
+    def _connection_update(self, **kwargs) -> None:
+        # language=rst
+        """
+        Hebbian learning rule for ``Connection`` subclass of ``AbstractConnection``
+        class.
+        """
 
-            # Add polarities back to feature after updates
-            if self.enforce_polarity:
-                self.feature_value = torch.abs(self.feature_value)
+        # Add polarities back to feature after updates
+        if self.enforce_polarity:
+            self.feature_value = torch.abs(self.feature_value)
 
-            batch_size = self.source.batch_size
+        batch_size = self.source.batch_size
 
-            source_s = self.source.s.view(batch_size, -1).unsqueeze(2).float()
-            source_x = self.source.x.view(batch_size, -1).unsqueeze(2)
-            target_s = self.target.s.view(batch_size, -1).unsqueeze(1).float()
-            target_x = self.target.x.view(batch_size, -1).unsqueeze(1)
+        source_s = self.source.s.view(batch_size, -1).unsqueeze(2).float()
+        source_x = self.source.x.view(batch_size, -1).unsqueeze(2)
+        target_s = self.target.s.view(batch_size, -1).unsqueeze(1).float()
+        target_x = self.target.x.view(batch_size, -1).unsqueeze(1)
 
-            # Pre-synaptic update.
-            update = self.reduction(torch.bmm(source_s, target_x), dim=0)
-            self.feature_value += self.nu[0] * update
+        # Pre-synaptic update.
+        update = self.reduction(torch.bmm(source_s, target_x), dim=0)
+        self.feature_value += self.nu[0] * update
 
-            # Post-synaptic update.
-            update = self.reduction(torch.bmm(source_x, target_s), dim=0)
-            self.feature_value += self.nu[1] * update
+        # Post-synaptic update.
+        update = self.reduction(torch.bmm(source_x, target_s), dim=0)
+        self.feature_value += self.nu[1] * update
 
-            # Add polarities back to feature after updates
-            if self.enforce_polarity:
-                self.feature_value = self.feature_value * self.polarities
+        # Add polarities back to feature after updates
+        if self.enforce_polarity:
+            self.feature_value = self.feature_value * self.polarities
 
-            super().update()
+        super().update()
 
-        def reset_state_variables(self):
-            return
+    def reset_state_variables(self):
+        return
 
 
 class MSTDP(MCC_LearningRule):

From 6f3c5a68151c745574a292c654a392805b27c6e7 Mon Sep 17 00:00:00 2001
From: christopher-earl <cearl@umass.edu>
Date: Sun, 26 Jul 2026 21:45:25 +0900
Subject: [PATCH 07/14] Test cases and minor bug fixes

---
 bindsnet/learning/MCC_learning.py     |   4 +-
 bindsnet/network/topology_features.py |   2 +
 test/network/test_connections.py      | 408 ++++++++++++++++++++++++++
 3 files changed, 413 insertions(+), 1 deletion(-)

diff --git a/bindsnet/learning/MCC_learning.py b/bindsnet/learning/MCC_learning.py
index 660bd538c..9c2a7b781 100644
--- a/bindsnet/learning/MCC_learning.py
+++ b/bindsnet/learning/MCC_learning.py
@@ -315,6 +315,7 @@ def __init__(
         self,
         connection: AbstractMulticompartmentConnection,
         feature_value: Union[torch.Tensor, float, int],
+        range: Optional[Sequence[float]] = None,
         nu: Optional[Union[float, Sequence[float]]] = None,
         reduction: Optional[callable] = None,
         decay: float = 0.0,
@@ -334,6 +335,7 @@ def __init__(
         super().__init__(
             connection=connection,
             feature_value=feature_value,
+            range=[-1, +1] if range is None else range,
             nu=nu,
             reduction=reduction,
             decay=decay,
@@ -344,7 +346,7 @@ def __init__(
             self.source.traces and self.target.traces
         ), "Both pre- and post-synaptic nodes must record spike traces."
 
-        if isinstance(MulticompartmentConnection):
+        if isinstance(connection, MulticompartmentConnection):
             self.update = self._connection_update
             self.feature_value = feature_value
         # elif isinstance(connection, Conv2dConnection):
diff --git a/bindsnet/network/topology_features.py b/bindsnet/network/topology_features.py
index f6126eba2..3cfc5a672 100644
--- a/bindsnet/network/topology_features.py
+++ b/bindsnet/network/topology_features.py
@@ -84,6 +84,7 @@ def __init__(
         from ..learning.MCC_learning import (
             NoOp,
             PostPre,
+            Hebbian,
             MSTDP,
             MSTDPET,
         )
@@ -91,6 +92,7 @@ def __init__(
         supported_rules = [
             NoOp,
             PostPre,
+            Hebbian,
             MSTDP,
             MSTDPET,
         ]
diff --git a/test/network/test_connections.py b/test/network/test_connections.py
index 24f2333e8..416855359 100644
--- a/test/network/test_connections.py
+++ b/test/network/test_connections.py
@@ -1,4 +1,5 @@
 import torch
+import math
 
 from bindsnet.learning import (
     MSTDP,
@@ -12,6 +13,8 @@
 from bindsnet.network import Network
 from bindsnet.network.nodes import Input, LIFNodes, SRM0Nodes
 from bindsnet.network.topology import *
+import bindsnet.learning.MCC_learning as mcc
+import bindsnet.network.topology_features as tf
 
 
 class TestConnection:
@@ -164,7 +167,412 @@ def test_weights(self, conn_type, shape_a, shape_b, shape_w, *args, **kwargs):
                     )
 
 
+class TestMultiCompartmentConnection:
+
+    def __init__(self):
+        self.device = torch.device("cpu")
+        print(f"Using device '{self.device}' for the MCC test")
+
+    # ----------------------------------------------------------------------- #
+    # Helpers                                                                 #
+    # ----------------------------------------------------------------------- #
+
+    def _make_mcc(self, pipeline, src_n, tgt_n, batch=1, sparse_compute=False):
+        """Build (and prime) a standalone MCC: Input(src_n) -> LIFNodes(tgt_n)."""
+        src = Input(n=src_n, traces=True)
+        tgt = LIFNodes(n=tgt_n, traces=True)
+        # batch_size is None until a Network sets it; the learning rules need it.
+        src.batch_size = batch
+        tgt.batch_size = batch
+        conn = MulticompartmentConnection(
+            source=src,
+            target=tgt,
+            device=self.device,
+            pipeline=pipeline,
+            sparse_compute=sparse_compute,
+        )
+        conn.dt = 1.0  # not set until added to a Network; rules read connection.dt
+        return conn
+
+    def _reference_expansion(self, pipeline, s, tgt_n):
+        """Pre-collapse pipeline features """
+        b, src = s.shape
+        x = s.view(b, src, 1).expand(b, src, tgt_n).clone().float()
+        for f in pipeline:
+            op = getattr(f, "op", "mul")
+            if isinstance(f, tf.Degradation):
+                v = (
+                    f.degrade_function(f.value)
+                    if f.degrade_function is not None
+                    else f.value
+                )
+            else:
+                v = f.value
+            if torch.is_tensor(v):
+                v = v.float()
+            if op == "mul":
+                x = x * v
+            elif op == "add":
+                x = x + v
+            else:  # "sub"
+                x = x - v
+        return x.sum(dim=1)
+
+    def _learning_conn(self, rule, w0, nu, rng=(-1.0, 1.0)):
+        """MCC with a single learnable Weight; returns (connection, feature)."""
+        src_n, tgt_n = w0.shape
+        conn = self._make_mcc(
+            [tf.Weight(name="w", value=w0.clone(), learning_rule=rule, nu=nu, range=rng)],
+            src_n,
+            tgt_n,
+            batch=1,
+        )
+        return conn, conn.pipeline[0]
+
+    def _mstdp_seq(self):
+        return [
+            (torch.tensor([1.0, 0.0, 0.0]), torch.tensor([0.0, 0.0]), 0.0),
+            (torch.tensor([0.0, 0.0, 0.0]), torch.tensor([1.0, 0.0]), 1.0),
+            (torch.tensor([0.0, 0.0, 0.0]), torch.tensor([0.0, 0.0]), 1.0),
+            (torch.tensor([0.0, 0.0, 0.0]), torch.tensor([0.0, 0.0]), 1.0),
+        ]
+
+    def _mstdp_reference(self, w0, seq, nu0, dt, tc_plus=20.0, tc_minus=20.0, rng=(-1.0, 1.0)):
+        w = w0.clone().float()
+        src_n, tgt_n = w.shape
+        p_plus, p_minus = torch.zeros(src_n), torch.zeros(tgt_n)
+        elig = torch.zeros(src_n, tgt_n)
+        dp, dm = math.exp(-dt / tc_plus), math.exp(-dt / tc_minus)
+        for src_s, tgt_s, reward in seq:
+            w = w + nu0 * reward * elig
+            p_plus = dp * p_plus + src_s
+            p_minus = dm * p_minus - tgt_s
+            elig = torch.outer(p_plus, tgt_s) + torch.outer(src_s, p_minus)
+            w = torch.clamp(w, rng[0], rng[1])
+        return w
+
+    def _mstdpet_reference(
+        self, w0, seq, nu0, dt, tc_plus=20.0, tc_minus=20.0, tc_e=25.0, rng=(-1.0, 1.0)
+    ):
+        w = w0.clone().float()
+        src_n, tgt_n = w.shape
+        p_plus, p_minus = torch.zeros(src_n), torch.zeros(tgt_n)
+        elig = torch.zeros(src_n, tgt_n)
+        elig_tr = torch.zeros(src_n, tgt_n)
+        for src_s, tgt_s, reward in seq:
+            elig_tr = elig_tr * math.exp(-dt / tc_e) + elig / tc_e
+            w = w + nu0 * dt * reward * elig_tr
+            p_plus = p_plus * math.exp(-dt / tc_plus) + src_s
+            p_minus = p_minus * math.exp(-dt / tc_minus) - tgt_s
+            elig = torch.outer(p_plus, tgt_s) + torch.outer(src_s, p_minus)
+            w = torch.clamp(w, rng[0], rng[1])
+        return w
+
+    # ----------------------------------------------------------------------- #
+    # Individual feature outputs                                              #
+    # ----------------------------------------------------------------------- #
+
+    def test_weight_feature_output(self):
+        s = torch.tensor([[1.0, 0.0, 1.0], [0.0, 1.0, 1.0]])
+        w = torch.tensor([[0.1, 0.2], [0.3, 0.4], [0.5, 0.6]])
+        conn = self._make_mcc([tf.Weight(name="w", value=w.clone())], 3, 2, batch=2)
+        assert torch.allclose(conn.compute(s), s @ w, atol=1e-6)
+
+    def test_mask_feature_output(self):
+        s = torch.tensor([[1.0, 1.0, 1.0]])
+        w = torch.tensor([[0.1, 0.2], [0.3, 0.4], [0.5, 0.6]])
+        mask = torch.tensor([[True, False], [True, True], [False, True]])
+        conn = self._make_mcc(
+            [tf.Weight(name="w", value=w.clone()), tf.Mask(name="m", value=mask.clone())],
+            3,
+            2,
+        )
+        assert torch.allclose(conn.compute(s), s @ (w * mask.float()), atol=1e-6)
+
+    def test_bias_feature_output(self):
+        # Bias is additive per-synapse; after the source-sum it adds bias.sum(0).
+        s = torch.tensor([[1.0, 0.0, 1.0], [1.0, 1.0, 0.0]])
+        w = torch.tensor([[0.1, 0.2], [0.3, 0.4], [0.5, 0.6]])
+        bias = torch.tensor([[0.5, -0.5], [0.1, 0.2], [0.0, 1.0]])
+        conn = self._make_mcc(
+            [tf.Weight(name="w", value=w.clone()), tf.Bias(name="b", value=bias.clone())],
+            3,
+            2,
+            batch=2,
+        )
+        assert torch.allclose(conn.compute(s), s @ w + bias.sum(0), atol=1e-6)
+
+    def test_intensity_feature_output(self):
+        # Intensity's value is a per-synapse [src, tgt] tensor (a constant 2.0 here).
+        s = torch.tensor([[1.0, 0.0, 1.0]])
+        w = torch.tensor([[0.1, 0.2], [0.3, 0.4], [0.5, 0.6]])
+        intensity = torch.full((3, 2), 2.0)
+        conn = self._make_mcc(
+            [
+                tf.Weight(name="w", value=w.clone()),
+                tf.Intensity(name="i", value=intensity.clone(), range=(-5.0, 5.0)),
+            ],
+            3,
+            2,
+        )
+        assert torch.allclose(conn.compute(s), s @ (w * intensity), atol=1e-6)
+
+    def test_degradation_feature_output(self):
+        # Degradation subtracts degrade_function(value) per-synapse -> -sum(0).
+        s = torch.tensor([[1.0, 1.0, 0.0]])
+        w = torch.tensor([[0.4, 0.2], [0.3, 0.4], [0.5, 0.6]])
+        deg = torch.tensor([[0.2, 0.4], [0.6, 0.8], [0.1, 0.3]])
+        conn = self._make_mcc(
+            [
+                tf.Weight(name="w", value=w.clone()),
+                tf.Degradation(
+                    name="d", value=deg.clone(), degrade_function=lambda v: v * 0.5
+                ),
+            ],
+            3,
+            2,
+        )
+        assert torch.allclose(conn.compute(s), s @ w - (0.5 * deg).sum(0), atol=1e-6)
+
+    def test_probability_feature_deterministic_bounds(self):
+        # bernoulli(1) == 1 (always passes); bernoulli(0) == 0 (always blocked).
+        s = torch.tensor([[1.0, 0.0, 1.0], [1.0, 1.0, 1.0]])
+        w = torch.tensor([[0.1, 0.2], [0.3, 0.4], [0.5, 0.6]])
+        passes = self._make_mcc(
+            [
+                tf.Probability(name="p", value=torch.ones(3, 2)),
+                tf.Weight(name="w", value=w.clone()),
+            ],
+            3,
+            2,
+            batch=2,
+        )
+        assert torch.allclose(passes.compute(s), s @ w, atol=1e-6)
+        blocked = self._make_mcc(
+            [
+                tf.Probability(name="p", value=torch.zeros(3, 2)),
+                tf.Weight(name="w", value=w.clone()),
+            ],
+            3,
+            2,
+            batch=2,
+        )
+        assert torch.allclose(blocked.compute(s), torch.zeros(2, 2), atol=1e-6)
+
+    def test_adaptation_features_output(self):
+        for cls in (tf.AdaptationBaseSynapsHistory, tf.AdaptationBaseOtherSynaps):
+            src_n, tgt_n = 3, 2
+            s = torch.tensor([[1.0, 0.0, 1.0]])
+            feat = cls(
+                name="a",
+                value=torch.zeros(src_n, tgt_n),
+                ann_values=[torch.zeros(1, 1), torch.zeros(1, 1)],
+            )
+            conn = self._make_mcc([feat], src_n, tgt_n, batch=1)
+            out = conn.compute(s)
+            assert torch.allclose(feat.value.float(), torch.ones(src_n, tgt_n))
+            assert torch.allclose(out, s @ torch.ones(src_n, tgt_n), atol=1e-6)
+
+    # ----------------------------------------------------------------------- #
+    # Combined pipelines == pre-collapse expansion                            #
+    # ----------------------------------------------------------------------- #
+
+    def test_pipeline_matches_expansion(self):
+        torch.manual_seed(0)
+        src_n, tgt_n, batch = 4, 3, 2
+        s = (torch.rand(batch, src_n) > 0.4).float()
+        w = torch.randn(src_n, tgt_n)
+        w2 = torch.randn(src_n, tgt_n)
+        mask = torch.rand(src_n, tgt_n) > 0.5
+        bias = torch.randn(src_n, tgt_n) * 0.2
+        deg = torch.rand(src_n, tgt_n)
+
+        pipelines = {
+            "weight": [tf.Weight(name="w", value=w.clone())],
+            "weight+mask": [
+                tf.Weight(name="w", value=w.clone()),
+                tf.Mask(name="m", value=mask.clone()),
+            ],
+            "weight+bias": [
+                tf.Weight(name="w", value=w.clone()),
+                tf.Bias(name="b", value=bias.clone()),
+            ],
+            "weight+bias+degradation": [
+                tf.Weight(name="w", value=w.clone()),
+                tf.Bias(name="b", value=bias.clone()),
+                tf.Degradation(
+                    name="d", value=deg.clone(), degrade_function=lambda v: v * 0.3
+                ),
+            ],
+            "weight+intensity+bias": [
+                tf.Weight(name="w", value=w.clone()),
+                tf.Intensity(
+                    name="i", value=torch.full((src_n, tgt_n), 1.5), range=(-5.0, 5.0)
+                ),
+                tf.Bias(name="b", value=bias.clone()),
+            ],
+            "weight,bias,weight,bias": [
+                tf.Weight(name="w", value=w.clone()),
+                tf.Bias(name="b", value=bias.clone()),
+                tf.Weight(name="w2", value=w2.clone()),
+                tf.Bias(name="b2", value=(bias * 0.5).clone()),
+            ],
+        }
+
+        for name, pipe in pipelines.items():
+            conn = self._make_mcc(pipe, src_n, tgt_n, batch=batch)
+            out = conn.compute(s)
+            ref = self._reference_expansion(pipe, s, tgt_n)
+            assert torch.allclose(
+                out, ref, atol=1e-5
+            ), f"{name}: {(out - ref).abs().max()}"
+
+    # ----------------------------------------------------------------------- #
+    # Sparse activity / sparse_compute == dense                               #
+    # ----------------------------------------------------------------------- #
+
+    def test_sparse_compute_matches_dense(self):
+        torch.manual_seed(1)
+        src_n, tgt_n, batch = 6, 4, 3
+        w = torch.randn(src_n, tgt_n)
+        bias = torch.randn(src_n, tgt_n) * 0.2
+        deg = torch.rand(src_n, tgt_n)
+
+        def pipes():
+            return [
+                [tf.Weight(name="w", value=w.clone())],
+                [
+                    tf.Weight(name="w", value=w.clone()),
+                    tf.Bias(name="b", value=bias.clone()),
+                ],
+                [
+                    tf.Weight(name="w", value=w.clone()),
+                    tf.Degradation(
+                        name="d", value=deg.clone(), degrade_function=lambda v: v * 0.4
+                    ),
+                ],
+            ]
+
+        spike_sets = [
+            (torch.rand(batch, src_n) > 0.5).float(),
+            torch.zeros(batch, src_n),  # empty-spike edge case
+        ]
+        for dense_pipe, sparse_pipe in zip(pipes(), pipes()):
+            for s in spike_sets:
+                dense = self._make_mcc(dense_pipe, src_n, tgt_n, batch=batch).compute(s)
+                sparse = self._make_mcc(
+                    sparse_pipe, src_n, tgt_n, batch=batch, sparse_compute=True
+                ).compute(s)
+                assert torch.allclose(dense, sparse, atol=1e-5)
+
+    # ----------------------------------------------------------------------- #
+    # MCC learning rules                                                      #
+    # ----------------------------------------------------------------------- #
+
+    def test_noop_leaves_weight_unchanged(self):
+        w0 = torch.rand(3, 2)
+        conn, feat = self._learning_conn(mcc.NoOp, w0, nu=(0.1, 0.1))
+        conn.source.s = torch.ones(1, 3)
+        conn.target.s = torch.ones(1, 2)
+        feat.learning_rule.update(reward=1.0)
+        assert torch.allclose(feat.value, w0)
+
+    def test_postpre_predictable(self):
+        # PostPre: dW = -nu0 * outer(src_s, tgt_x) + nu1 * outer(src_x, tgt_s).
+        w0 = torch.tensor([[0.1, 0.2], [0.3, 0.4], [0.0, -0.2]])
+        conn, feat = self._learning_conn(mcc.PostPre, w0, nu=(0.1, 0.2))
+        conn.source.s = torch.tensor([[1.0, 0.0, 1.0]])
+        conn.target.s = torch.tensor([[1.0, 1.0]])
+        conn.source.x = torch.tensor([[0.3, 0.7, 0.2]])
+        conn.target.x = torch.tensor([[0.5, 0.4]])
+        feat.learning_rule.update()
+        dw = -0.1 * torch.outer(conn.source.s[0], conn.target.x[0]) + 0.2 * torch.outer(
+            conn.source.x[0], conn.target.s[0]
+        )
+        expected = torch.clamp(w0 + dw, -1.0, 1.0)
+        assert torch.allclose(feat.value, expected, atol=1e-6)
+        assert not torch.allclose(feat.value, w0)  # sanity: it actually changed
+
+    def test_learning_respects_range_clamp(self):
+        # Post-only potentiation of +5 per synapse must clamp to the range max (1.0).
+        w0 = torch.full((2, 2), 0.9)
+        conn, feat = self._learning_conn(mcc.PostPre, w0, nu=(0.0, 5.0), rng=(-1.0, 1.0))
+        conn.source.s = torch.zeros(1, 2)
+        conn.target.s = torch.ones(1, 2)
+        conn.source.x = torch.ones(1, 2)
+        conn.target.x = torch.zeros(1, 2)
+        feat.learning_rule.update()
+        assert torch.allclose(feat.value, torch.ones(2, 2))
+
+    def test_mstdp_predictable(self):
+        w0 = torch.zeros(3, 2)
+        conn, feat = self._learning_conn(mcc.MSTDP, w0, nu=(0.5, 0.0))
+        seq = self._mstdp_seq()
+        for src_v, tgt_v, reward in seq:
+            conn.source.s = src_v.unsqueeze(0)
+            conn.target.s = tgt_v.unsqueeze(0)
+            feat.learning_rule.update(reward=reward)
+        expected = self._mstdp_reference(w0, seq, nu0=0.5, dt=1.0)
+        assert torch.allclose(feat.value, expected, atol=1e-5)
+        assert not torch.allclose(feat.value, w0)
+
+    def test_mstdpet_predictable(self):
+        w0 = torch.zeros(3, 2)
+        conn, feat = self._learning_conn(mcc.MSTDPET, w0, nu=(0.5, 0.5))
+        seq = self._mstdp_seq()
+        for src_v, tgt_v, reward in seq:
+            conn.source.s = src_v.unsqueeze(0)
+            conn.target.s = tgt_v.unsqueeze(0)
+            feat.learning_rule.update(reward=reward)
+        expected = self._mstdpet_reference(w0, seq, nu0=0.5, dt=1.0)
+        assert torch.allclose(feat.value, expected, atol=1e-5)
+        assert not torch.allclose(feat.value, w0)
+
+    def test_hebbian_predictable(self):
+        # Hebbian: dW = nu0 * outer(src_s, tgt_x) + nu1 * outer(src_x, tgt_s), then
+        # clamped. Both pre- and post-synaptic terms are positive (contrast PostPre,
+        # which subtracts the pre term).
+        w0 = torch.tensor([[0.1, 0.2], [0.3, 0.4], [0.0, -0.2]])
+        conn, feat = self._learning_conn(mcc.Hebbian, w0, nu=(0.1, 0.2))
+        conn.source.s = torch.tensor([[1.0, 0.0, 1.0]])
+        conn.target.s = torch.tensor([[1.0, 1.0]])
+        conn.source.x = torch.tensor([[0.3, 0.7, 0.2]])
+        conn.target.x = torch.tensor([[0.5, 0.4]])
+        feat.learning_rule.update()
+        dw = 0.1 * torch.outer(conn.source.s[0], conn.target.x[0]) + 0.2 * torch.outer(
+            conn.source.x[0], conn.target.s[0]
+        )
+        expected = torch.clamp(w0 + dw, -1.0, 1.0)
+        assert torch.allclose(feat.value, expected, atol=1e-6)
+        assert not torch.allclose(feat.value, w0)  # sanity: it actually changed
+
+
 if __name__ == "__main__":
+    # MulticompartmentConnection + MCC learning-rule tests.
+    mcc_tester = TestMultiCompartmentConnection()
+    mcc_tests = [
+        mcc_tester.test_weight_feature_output,
+        mcc_tester.test_mask_feature_output,
+        mcc_tester.test_bias_feature_output,
+        mcc_tester.test_intensity_feature_output,
+        mcc_tester.test_degradation_feature_output,
+        mcc_tester.test_probability_feature_deterministic_bounds,
+        mcc_tester.test_meanfield_not_implemented,
+        mcc_tester.test_adaptation_features_output,
+        mcc_tester.test_pipeline_matches_expansion,
+        mcc_tester.test_sparse_compute_matches_dense,
+        mcc_tester.test_noop_leaves_weight_unchanged,
+        mcc_tester.test_postpre_predictable,
+        mcc_tester.test_learning_respects_range_clamp,
+        mcc_tester.test_mstdp_predictable,
+        mcc_tester.test_mstdpet_predictable,
+        mcc_tester.test_hebbian_predictable,
+    ]
+    for mcc_test in mcc_tests:
+        mcc_test()
+        print(f"  PASSED: {mcc_test.__name__}")
+    print(f"All {len(mcc_tests)} MulticompartmentConnection tests passed.")
+
     tester = TestConnection()
 
     # tester.test_transfer()

From a81c05dbb2b05baa556b7bb4a41d76c10ef4b1e0 Mon Sep 17 00:00:00 2001
From: christopher-earl <cearl@umass.edu>
Date: Sun, 26 Jul 2026 23:04:16 +0900
Subject: [PATCH 08/14] Benchmark tests

---
 .../_bench_common.py                          | 137 +++++++++++++
 .../foldable_pipelines.py                     |   7 +
 .../run_both.py                               |   9 +
 .../sparse_compute.py                         |   8 +
 examples/stress_test/example_network.py       | 185 ++++++++++++++++++
 5 files changed, 346 insertions(+)
 create mode 100644 examples/benchmark/sparse_compute and foldable pipeline/_bench_common.py
 create mode 100644 examples/benchmark/sparse_compute and foldable pipeline/foldable_pipelines.py
 create mode 100644 examples/benchmark/sparse_compute and foldable pipeline/run_both.py
 create mode 100644 examples/benchmark/sparse_compute and foldable pipeline/sparse_compute.py
 create mode 100644 examples/stress_test/example_network.py

diff --git a/examples/benchmark/sparse_compute and foldable pipeline/_bench_common.py b/examples/benchmark/sparse_compute and foldable pipeline/_bench_common.py
new file mode 100644
index 000000000..564de6591
--- /dev/null
+++ b/examples/benchmark/sparse_compute and foldable pipeline/_bench_common.py	
@@ -0,0 +1,137 @@
+"""
+Shared machinery for the ExampleNetwork MCC benchmarks (imported by
+``foldable_pipelines.py``, ``sparse_compute.py`` and ``both.py``).
+
+Each benchmark reports the %-time speedup of one MCC configuration over another by
+timing the *same* ExampleNetwork under different ``compute`` modes:
+
+  * ``expansion``   -- the pre-optimization path, reconstructed here as a
+                       monkeypatch: materialize ``[batch, src, tgt]``, apply every
+                       feature elementwise, then sum over source. (This path was
+                       removed from the code when the fold landed, so we rebuild it
+                       to serve as the baseline.)
+  * ``fold``        -- the current folded path: ``out = s @ A + B.sum(0)``.
+  * ``fold_sparse`` -- the fold plus activity-sparse compute (``sparse_compute``):
+                       read only the weight rows of source neurons that spiked.
+
+Speedup is wall-time reduction: ``(t_baseline - t_new) / t_baseline * 100``
+(positive = faster). Learning is disabled so we time the pure forward path (the
+part these optimizations affect).
+"""
+
+import os
+import statistics
+import sys
+import time
+
+_HERE = os.path.dirname(os.path.abspath(__file__))
+_ROOT = os.path.dirname(os.path.dirname(_HERE))  # repo root (for ``bindsnet``)
+_STRESS = os.path.join(_ROOT, "stress_test")  # for ``example_network``
+for _p in (_ROOT, _STRESS):
+    if _p not in sys.path:
+        sys.path.insert(0, _p)
+
+import torch
+
+from bindsnet.network.topology import MulticompartmentConnection
+from bindsnet.network.topology_features import Degradation, Probability
+from example_network import ExampleNetwork
+
+# ExampleNetwork sizes per device: 20k excitatory neurons on GPU (where the fold
+# shines), a smaller net on CPU so the baseline finishes in reasonable time.
+GPU_CONFIG = dict(in_size=100, exc_size=20_000, inh_size=2_000)
+CPU_CONFIG = dict(in_size=100, exc_size=2_000, inh_size=200)
+GPU_TIME, CPU_TIME = 50, 20
+REPS, WARMUP = 5, 2
+
+
+def _expansion_compute(self, s):
+    """The pre-fold ``[batch, src, tgt]`` expansion path (baseline)."""
+    s = s.view(s.size(0), self.source.n)
+    cs = s.view(s.size(0), self.source.n, 1).repeat(1, 1, self.target.n)
+    for f in self.pipeline:
+        op = getattr(f, "op", "mul")
+        if isinstance(f, Probability):
+            v = torch.bernoulli(f.value)
+        elif isinstance(f, Degradation):
+            v = f.degrade_function(f.value) if f.degrade_function is not None else f.value
+        else:
+            v = f.value
+        if op == "mul":
+            cs = cs * v
+        elif op == "add":
+            cs = cs + v
+        else:
+            cs = cs - v
+    out = cs.sum(1)
+    if getattr(self, "traces", False):
+        self.activity = out
+    if out.size() != torch.Size([s.size(0)] + self.target.shape):
+        return out.view(s.size(0), *self.target.shape)
+    return out
+
+
+def _set_mode(net, mode):
+    for c in net.connections.values():
+        if isinstance(c, MulticompartmentConnection):
+            if mode == "expansion":
+                c.compute = _expansion_compute.__get__(c)  # instance override
+                c.sparse_compute = False
+            else:
+                c.__dict__.pop("compute", None)  # restore the class (fold) method
+                c.sparse_compute = mode == "fold_sparse"
+
+
+def _bench(net, inputs, T, device, modes):
+    cuda = device.startswith("cuda")
+    net.train(False)  # forward-only: time the compute path the optimizations touch
+    for m in modes:  # warmup each mode
+        _set_mode(net, m)
+        for _ in range(WARMUP):
+            net.reset_state_variables()
+            net.run(inputs=inputs, time=T)
+    if cuda:
+        torch.cuda.synchronize()
+    samples = {m: [] for m in modes}
+    for _ in range(REPS):  # interleave modes each rep to counter thermal drift
+        for m in modes:
+            _set_mode(net, m)
+            net.reset_state_variables()
+            if cuda:
+                torch.cuda.synchronize()
+            t0 = time.perf_counter()
+            net.run(inputs=inputs, time=T)
+            if cuda:
+                torch.cuda.synchronize()
+            samples[m].append(time.perf_counter() - t0)
+    return {m: statistics.median(v) * 1e3 for m, v in samples.items()}
+
+
+def run_speedup(baseline_mode, test_mode, technique):
+    """Build the ExampleNetwork on CPU and GPU, time both modes, print the speedup."""
+    print("=" * 72)
+    print(f"{technique}")
+    print(f"  (%-time speedup of '{test_mode}' vs baseline '{baseline_mode}')")
+    print("=" * 72)
+
+    devices = ["cpu"] + (["cuda"] if torch.cuda.is_available() else [])
+    if not torch.cuda.is_available():
+        print("[note] CUDA not available -> reporting CPU only.\n")
+
+    for dev in devices:
+        cfg = GPU_CONFIG if dev == "cuda" else CPU_CONFIG
+        T = GPU_TIME if dev == "cuda" else CPU_TIME
+        net = ExampleNetwork(device=dev, **cfg)
+        inputs = net.make_input(T)
+        res = _bench(net, inputs, T, dev, [baseline_mode, test_mode])
+        base, new = res[baseline_mode], res[test_mode]
+        speedup = (base - new) / base * 100.0
+        print(
+            f"  [{dev.upper():4s}] exc={cfg['exc_size']:>6d}  time={T:>3d}  |  "
+            f"{baseline_mode}={base:8.2f} ms   {test_mode}={new:8.2f} ms   "
+            f"|  speedup = {speedup:+6.1f}% time"
+        )
+        del net
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+    print()
diff --git a/examples/benchmark/sparse_compute and foldable pipeline/foldable_pipelines.py b/examples/benchmark/sparse_compute and foldable pipeline/foldable_pipelines.py
new file mode 100644
index 000000000..ea43adfbe
--- /dev/null
+++ b/examples/benchmark/sparse_compute and foldable pipeline/foldable_pipelines.py	
@@ -0,0 +1,7 @@
+from _bench_common import run_speedup
+
+### Benchmark: foldable pipelines ###
+# Measures the speedup of the folded MultiCompartmentConnection compute (``out = s @ A + B.sum(0)``) over
+# the pre-fold ``[batch, src, tgt]`` expansion path, on the ExampleNetwork
+if __name__ == "__main__":
+    run_speedup(baseline_mode="expansion", test_mode="fold", technique="Foldable pipelines")
diff --git a/examples/benchmark/sparse_compute and foldable pipeline/run_both.py b/examples/benchmark/sparse_compute and foldable pipeline/run_both.py
new file mode 100644
index 000000000..0752da254
--- /dev/null
+++ b/examples/benchmark/sparse_compute and foldable pipeline/run_both.py	
@@ -0,0 +1,9 @@
+from _bench_common import run_speedup
+
+# Run both sparse_compute.py and foldable_pipelines.py
+if __name__ == "__main__":
+    run_speedup(
+        baseline_mode="expansion",
+        test_mode="fold_sparse",
+        technique="Foldable pipelines + sparse compute (combined)",
+    )
diff --git a/examples/benchmark/sparse_compute and foldable pipeline/sparse_compute.py b/examples/benchmark/sparse_compute and foldable pipeline/sparse_compute.py
new file mode 100644
index 000000000..3a4a6f733
--- /dev/null
+++ b/examples/benchmark/sparse_compute and foldable pipeline/sparse_compute.py	
@@ -0,0 +1,8 @@
+from _bench_common import run_speedup
+
+### Benchmark: sparse compute ###
+# Measures the speedup of activity-sparse compute (``sparse_compute=True`` -- read
+# only the weight rows of source neurons that spiked) over the dense folded compute,
+# on the ExampleNetwork (CPU and GPU).
+if __name__ == "__main__":
+    run_speedup(baseline_mode="fold", test_mode="fold_sparse", technique="Sparse compute")
diff --git a/examples/stress_test/example_network.py b/examples/stress_test/example_network.py
new file mode 100644
index 000000000..7562a2736
--- /dev/null
+++ b/examples/stress_test/example_network.py
@@ -0,0 +1,185 @@
+"""
+ExampleNetwork -- a large, sparse, recurrent ``MulticompartmentConnection`` (MCC)
+stress workload.
+
+Topology: ``Input(I) -> EXC_LIF <-> INH_LIF``. Four single-``Weight`` MCC.
+The ``I -> EXC`` connection also carries an ``MSTDP`` learning rule.
+
+Run it directly to stress the simulator:
+
+    python examples/stress_test/example_network.py --device cuda --exc 20000 --time 50
+    python examples/stress_test/example_network.py --device cpu  --exc 2000  --time 20
+"""
+
+import argparse
+import os
+import sys
+import time
+
+# Make ``bindsnet`` importable when this file is run as a standalone script.
+sys.path.insert(
+    0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+)
+
+import torch
+
+from bindsnet.learning.MCC_learning import MSTDP
+from bindsnet.network.network import Network
+from bindsnet.network.nodes import Input, LIFNodes
+from bindsnet.network.topology import MulticompartmentConnection
+from bindsnet.network.topology_features import Weight
+
+
+class ExampleNetwork(Network):
+    def __init__(
+        self,
+        device="cpu",
+        in_size=100,
+        exc_size=20_000,
+        inh_size=2_000,
+        batch_size=1,
+        i_to_exc_connectivity=0.15,
+        i_to_inh_connectivity=0.05,
+        inh_to_exc_connectivity=0.05,
+        exc_to_inh_connectivity=0.05,
+    ):
+        super().__init__()
+        self.device = device
+        self.in_size = in_size
+        self.exc_size = exc_size
+        self.inh_size = inh_size
+        self.batch_size = batch_size
+        self.i_to_exc_connectivity = i_to_exc_connectivity
+        self.i_to_inh_connectivity = i_to_inh_connectivity
+        self.inh_to_exc_connectivity = inh_to_exc_connectivity
+        self.exc_to_inh_connectivity = exc_to_inh_connectivity
+        self.build()
+
+    def _sparse_weight(self, rows, cols, connectivity, sign=1.0):
+        w = sign * torch.rand(rows, cols, device=self.device)
+        keep = torch.rand(rows, cols, device=self.device) > (1 - connectivity)
+        return w * keep
+
+    def build(self):
+        device = self.device
+        self.add_layer(layer=Input(self.in_size), name="I")
+        self.add_layer(layer=LIFNodes(self.exc_size), name="EXC_LIF")
+        self.add_layer(layer=LIFNodes(self.inh_size), name="INH_LIF")
+        self.add_connection(
+            connection=MulticompartmentConnection(
+                source=self.layers["I"],
+                target=self.layers["EXC_LIF"],
+                device=device,
+                pipeline=[
+                    Weight(
+                        name="I_to_EXC_weight",
+                        value=self._sparse_weight(
+                            self.in_size, self.exc_size, self.i_to_exc_connectivity
+                        ),
+                        learning_rule=MSTDP,
+                        range=(0, 1),
+                    )
+                ],
+            ),
+            source="I",
+            target="EXC_LIF",
+        )
+        self.add_connection(
+            connection=MulticompartmentConnection(
+                source=self.layers["I"],
+                target=self.layers["INH_LIF"],
+                device=device,
+                pipeline=[
+                    Weight(
+                        name="I_to_INH_weight",
+                        value=self._sparse_weight(
+                            self.in_size, self.inh_size, self.i_to_inh_connectivity
+                        ),
+                    )
+                ],
+            ),
+            source="I",
+            target="INH_LIF",
+        )
+        self.add_connection(
+            connection=MulticompartmentConnection(
+                source=self.layers["INH_LIF"],
+                target=self.layers["EXC_LIF"],
+                device=device,
+                pipeline=[
+                    Weight(
+                        name="INH_to_EXC_weight",
+                        value=self._sparse_weight(
+                            self.inh_size,
+                            self.exc_size,
+                            self.inh_to_exc_connectivity,
+                            sign=-1.0,
+                        ),
+                    )
+                ],
+            ),
+            source="INH_LIF",
+            target="EXC_LIF",
+        )
+        self.add_connection(
+            connection=MulticompartmentConnection(
+                source=self.layers["EXC_LIF"],
+                target=self.layers["INH_LIF"],
+                device=device,
+                pipeline=[
+                    Weight(
+                        name="EXC_to_INH_weight",
+                        value=self._sparse_weight(
+                            self.exc_size, self.inh_size, self.exc_to_inh_connectivity
+                        ),
+                    )
+                ],
+            ),
+            source="EXC_LIF",
+            target="INH_LIF",
+        )
+        self.to(device)
+
+    def make_input(self, runtime):
+        # Poisson-ish random spike train into the input layer.
+        return {
+            "I": torch.rand(
+                runtime, self.batch_size, self.in_size, device=self.device
+            )
+            > 0.90
+        }
+
+
+if __name__ == "__main__":
+    p = argparse.ArgumentParser(description="Stress-test the ExampleNetwork.")
+    p.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu")
+    p.add_argument("--in-size", type=int, default=100)
+    p.add_argument("--exc", type=int, default=20_000)
+    p.add_argument("--inh", type=int, default=2_000)
+    p.add_argument("--time", type=int, default=50)
+    args = p.parse_args()
+
+    device = args.device
+    if device.startswith("cuda") and not torch.cuda.is_available():
+        print("[note] CUDA unavailable; falling back to CPU.")
+        device = "cpu"
+
+    net = ExampleNetwork(
+        device=device, in_size=args.in_size, exc_size=args.exc, inh_size=args.inh
+    )
+    net.train(False)  # forward-only stress (no learning)
+    inputs = net.make_input(args.time)
+
+    net.run(inputs=inputs, time=args.time)  # warmup
+    if device.startswith("cuda"):
+        torch.cuda.synchronize()
+    net.reset_state_variables()
+    t0 = time.perf_counter()
+    net.run(inputs=inputs, time=args.time)
+    if device.startswith("cuda"):
+        torch.cuda.synchronize()
+    ms = (time.perf_counter() - t0) * 1e3
+    print(
+        f"ExampleNetwork [{device}] in={args.in_size} exc={args.exc} inh={args.inh} "
+        f"time={args.time}: {ms:.1f} ms total ({ms / args.time:.3f} ms/step)"
+    )

From 79de6b4cffdea957c21d1fab76adb414e9d9e77f Mon Sep 17 00:00:00 2001
From: Hananel Hazan <Hananel@Hazan.org.il>
Date: Mon, 3 Aug 2026 20:22:13 -0400
Subject: [PATCH 09/14] black formater

---
 .claude/settings.json                         |   8 +
 .claude/skills/archive-log/SKILL.md           | 100 ++++
 .claude/skills/archive-search/SKILL.md        |  95 ++++
 .claude/skills/generate-leaderboard/SKILL.md  |  87 ++++
 .claude/skills/generate-report/SKILL.md       | 120 +++++
 .claude/skills/hpc-submit/SKILL.md            | 256 ++++++++++
 .claude/skills/intake-results/SKILL.md        | 101 ++++
 .claude/skills/log-entry/SKILL.md             | 205 ++++++++
 .claude/skills/pax-hpc/SKILL.md               | 292 +++++++++++
 .claude/skills/pax-hpc/references/intake.md   |  92 ++++
 .../skills/pax-hpc/references/new-cluster.md  | 125 +++++
 .../skills/pax-hpc/references/old-cluster.md  | 105 ++++
 .../skills/pax-hpc/references/resilience.md   | 457 ++++++++++++++++++
 .claude/skills/report/SKILL.md                |  53 ++
 .claude/skills/slurm-harvest/SKILL.md         |  38 ++
 .claude/skills/slurm-harvest/harvest_slurm.py | 189 ++++++++
 bindsnet/datasets/torchvision_wrapper.py      |   9 +-
 .../_bench_common.py                          |   6 +-
 .../foldable_pipelines.py                     |   4 +-
 .../sparse_compute.py                         |   4 +-
 examples/stress_test/example_network.py       |   4 +-
 test/network/test_connections.py              |  26 +-
 22 files changed, 2362 insertions(+), 14 deletions(-)
 create mode 100644 .claude/settings.json
 create mode 100644 .claude/skills/archive-log/SKILL.md
 create mode 100644 .claude/skills/archive-search/SKILL.md
 create mode 100644 .claude/skills/generate-leaderboard/SKILL.md
 create mode 100644 .claude/skills/generate-report/SKILL.md
 create mode 100644 .claude/skills/hpc-submit/SKILL.md
 create mode 100644 .claude/skills/intake-results/SKILL.md
 create mode 100644 .claude/skills/log-entry/SKILL.md
 create mode 100644 .claude/skills/pax-hpc/SKILL.md
 create mode 100644 .claude/skills/pax-hpc/references/intake.md
 create mode 100644 .claude/skills/pax-hpc/references/new-cluster.md
 create mode 100644 .claude/skills/pax-hpc/references/old-cluster.md
 create mode 100644 .claude/skills/pax-hpc/references/resilience.md
 create mode 100644 .claude/skills/report/SKILL.md
 create mode 100644 .claude/skills/slurm-harvest/SKILL.md
 create mode 100644 .claude/skills/slurm-harvest/harvest_slurm.py

diff --git a/.claude/settings.json b/.claude/settings.json
new file mode 100644
index 000000000..3050c4333
--- /dev/null
+++ b/.claude/settings.json
@@ -0,0 +1,8 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(curl -sL \"https://zenodo.org/api/records/20695116\")",
+      "Bash(python -c \"import sys,json; d=json.load\\(sys.stdin\\); print\\('doi:', d.get\\('doi'\\)\\); print\\('conceptdoi:', d.get\\('conceptdoi'\\)\\); print\\('title:', d.get\\('metadata',{}\\).get\\('title'\\)\\); print\\('conceptrecid:', d.get\\('conceptrecid'\\)\\)\")"
+    ]
+  }
+}
diff --git a/.claude/skills/archive-log/SKILL.md b/.claude/skills/archive-log/SKILL.md
new file mode 100644
index 000000000..a1416230c
--- /dev/null
+++ b/.claude/skills/archive-log/SKILL.md
@@ -0,0 +1,100 @@
+---
+name: archive-log
+description: Archives old entries from a per-task log file when it grows too large. Moves entries beyond the keep-last threshold to logs/archive/. Also use when the user says "archive the log", "trim the log", or "the log is too long".
+allowed-tools: Bash, Read, Edit
+---
+
+# Archive Old Log Entries
+
+## When to invoke
+
+- Manually, when a task log is getting unwieldy
+- Automatically, called by `/log-entry` when entry count > 10
+- Automatically, called by `/intake-results` Step 8
+- During migration from the old monolithic LOG.md
+
+## Step 1 -- Check the current state
+
+```bash
+# Count entries
+grep -c "^## " logs/LOG.<task>.md
+
+# See the oldest entries that will be archived
+grep "^## " logs/LOG.<task>.md | tail -10
+```
+
+## Step 2 -- Dry-run the archive script
+
+```bash
+python scripts/archive_log.py --task <task> --keep-last 5 --dry-run
+```
+
+Read the output carefully:
+- How many entries will be archived?
+- What is the archive filename?
+- What pointer line will be written into the active log?
+
+## Step 3 -- Execute
+
+```bash
+python scripts/archive_log.py --task <task> --keep-last 5
+```
+
+## Step 4 -- Verify
+
+```bash
+# Active log should now have 5 entries + 1 pointer line near the bottom
+grep -c "^## " logs/LOG.<task>.md
+grep "^> Archived entries" logs/LOG.<task>.md
+
+# Archive file should exist
+ls logs/archive/LOG.<task>.*.md
+```
+
+### Pinned block check
+
+```bash
+head -5 logs/LOG.<task>.md | grep "📌"
+```
+
+The `## 📌 Current Best` block must still be present at the top after archival.
+If it is missing, the `split_entries` function did not handle it correctly —
+restore from the archive and fix the script before continuing.
+
+## Step 5 -- Update global index
+
+Open `LOG.md` and confirm the row for `<task>` still points to `logs/LOG.<task>.md` (the active file -- not the archive).
+
+## Step 6 -- Rebuild the search index
+
+The archive script auto-triggers this, but if it didn't run, do it manually:
+
+```bash
+python scripts/build_search_index.py
+```
+
+This regenerates `logs/archive/INDEX.jsonl` with all entries (active + archived).
+
+## Step 7 -- Refresh STATUS
+
+The hot log just changed; regenerate the compact STATUS file (Claude's default
+read surface):
+
+```bash
+python scripts/generate_status.py --task <task>
+```
+
+## Archive file naming
+
+`logs/archive/LOG.<task>.<YYYY-MM-DD>.<seq>.md`
+
+The `<seq>` suffix (01, 02, ...) prevents collisions if you archive the same task multiple times in one day.
+
+## Pointer line format (written into the active log)
+
+```markdown
+> Archived entries 001--NNN -> logs/archive/LOG.<task>.<date>.<seq>.md
+```
+
+This line is always kept at the **bottom** of the active log, below all `## ` entries.
+Multiple pointer lines accumulate as more archives are created -- do not delete them.
diff --git a/.claude/skills/archive-search/SKILL.md b/.claude/skills/archive-search/SKILL.md
new file mode 100644
index 000000000..79ac3c828
--- /dev/null
+++ b/.claude/skills/archive-search/SKILL.md
@@ -0,0 +1,95 @@
+---
+name: archive-search
+description: Searches archived log entries across all tasks or a specific task. Use when the user says "what did we try before", "search the archive", "find old runs with X", or when the agent needs historical context beyond the 5 hot entries in the active log.
+allowed-tools: Bash, Read
+---
+
+# Search Archived Log Entries
+
+## Step 1 — Identify scope
+
+Determine: is this a single-task search or cross-task?
+
+```bash
+# List all archive files
+ls logs/archive/
+
+# For a specific task:
+ls logs/archive/LOG.<task>.*.md
+
+# Total archive size
+wc -l logs/archive/*.md | tail -1
+```
+
+## Step 1.5 — Check the search index (fast path)
+
+If the search can be answered by metadata alone (run number, date, metric, title keyword, task),
+use the structured index first:
+
+```bash
+python3 -c "
+import json
+for line in open('logs/archive/INDEX.jsonl'):
+    e = json.loads(line)
+    if '<keyword>' in e.get('title','').lower() or '<keyword>' in str(e.get('tags',[])).lower():
+        metric = f\"{e.get('key_metric','')}: {e.get('key_value','')}\" if e.get('key_value') else ''
+        print(f\"{e['date']}  Run {e.get('run','?'):>3}  [{e['task']}]  {e['title']}  {metric}\")
+"
+```
+
+For metric-based queries (e.g. "accuracy > 90%"):
+
+```bash
+python3 -c "
+import json
+for line in open('logs/archive/INDEX.jsonl'):
+    e = json.loads(line)
+    if e.get('key_value') is not None and e['key_value'] > 90:
+        print(f\"{e['date']}  Run {e.get('run','?'):>3}  [{e['task']}]  {e.get('key_metric','')}: {e['key_value']}\")
+"
+```
+
+If this returns sufficient results, you can skip the full-text grep in Steps 2-3.
+
+## Step 2 — Search
+
+```bash
+# Search for a keyword across all archives
+grep -l "<keyword>" logs/archive/*.md
+
+# Search within a specific task's archives
+grep -n "<keyword>" logs/archive/LOG.<task>.*.md
+
+# Find runs matching a config value (e.g. learning rate)
+grep -n "lr=1e-3\|learning_rate: 0.001" logs/archive/LOG.<task>.*.md
+
+# Find runs by approximate date range
+ls logs/archive/LOG.<task>.2025-0[1-3]*.md
+```
+
+## Step 3 — Read matching entries
+
+Once you have a filename and line number, read the surrounding context:
+
+```bash
+# Read a specific archive file
+cat logs/archive/LOG.<task>.<date>.<seq>.md
+
+# Or just the matching section (N lines around match)
+grep -A 20 "<keyword>" logs/archive/LOG.<task>.*.md | head -60
+```
+
+## Step 4 — Summarize findings
+
+Report to the user:
+- Which archive files matched
+- Run numbers and dates of matching entries
+- The key content (config, result, notes) from each match
+- Whether any matching run is a candidate to resume or replicate
+
+## Notes
+
+- Archive files are read-only. Never write to them.
+- If a keyword matches many entries, filter by date or metric value.
+- Active log entries (logs/LOG.<task>.md) are not searched here —
+  the agent can read those directly.
diff --git a/.claude/skills/generate-leaderboard/SKILL.md b/.claude/skills/generate-leaderboard/SKILL.md
new file mode 100644
index 000000000..782cc0be7
--- /dev/null
+++ b/.claude/skills/generate-leaderboard/SKILL.md
@@ -0,0 +1,87 @@
+---
+name: generate-leaderboard
+description: Reads the Current Best pinned block from every per-task log and produces a ranked leaderboard table across all tasks. Use when the user says "show me the leaderboard", "what is the best result per task", "update the summary table", or "what should I work on next".
+allowed-tools: Read, Write, Bash, Glob
+---
+
+# Generate Cross-Task Leaderboard
+
+## Step 1a — Fast path: aggregate run.json sidecars (preferred)
+
+Every promoted run has a flat `run.json`. Aggregate them without touching markdown:
+
+```bash
+python3 -c "
+import json, pathlib
+from collections import defaultdict
+best = defaultdict(lambda: (None, None))  # task -> (value, record)
+for p in pathlib.Path('experiments').rglob('run.json'):
+    try:
+        r = json.loads(p.read_text())
+    except Exception:
+        continue
+    task = r.get('task'); v = r.get('key_value'); km = r.get('key_metric')
+    if task is None or v is None: continue
+    # Lower-is-better for ppl/loss/mae/mse; higher-is-better otherwise
+    lower_better = km in {'test_ppl','val_ppl','mean_ppl','loss','mae','mse'}
+    cur_v, _ = best[task]
+    if cur_v is None or (lower_better and v < cur_v) or (not lower_better and v > cur_v):
+        best[task] = (v, r)
+for task, (v, r) in sorted(best.items()):
+    print(f\"{task:<30} {r.get('key_metric',''):<10} {v:>10}  run={r.get('run_id')}  {r.get('date','')}\")
+"
+```
+
+If a task has no sidecars, fall back to Step 1b.
+
+## Step 1b — Fallback: pinned blocks from STATUS / LOG
+
+```bash
+for f in logs/STATUS.*.md logs/LOG.*.md; do
+  echo "=== $f ==="
+  awk '/^## 📌 Current Best/{found=1} found{print} /^---/{if(found) exit}' "$f"
+  echo ""
+done
+```
+
+Prefer `logs/STATUS.*.md` — smaller and authoritative for the pinned block.
+
+## Step 2 — Build the leaderboard table
+
+For each task, extract the 🥇 row from its pinned block.
+Produce a single markdown table sorted by metric value (best first).
+Group tasks by metric type if they use different metrics.
+
+Output format:
+
+```markdown
+## Leaderboard — <date>
+
+### <metric group> (higher is better / lower is better)
+
+| Rank | Task | Run | Date | Metric | Value | Config |
+|------|------|-----|------|--------|-------|--------|
+| 1 | mnist_cnn | Run 057 | 2025-04-10 | acc | 98.3% | lr=1e-3 |
+| 2 | cifar10 | Run 031 | 2025-03-22 | acc | 94.1% | lr=3e-4 |
+...
+
+### No result yet
+
+Tasks with no quantitative result: <comma-separated list>
+```
+
+## Step 3 — Write output
+
+Save to `experiments/_leaderboard/leaderboard_<YYYY-MM-DD>.md`.
+Also print to stdout for immediate review.
+
+## Step 4 — Embed in PAPER.md (optional)
+
+If the user asks to update PAPER.md, replace the content under
+`## Summary` or `## Current Results` with the leaderboard table.
+Do not touch other sections.
+
+## Step 5 — Bi-directional sync
+
+No LOG entry needed for leaderboard generation unless results changed.
+If you updated PAPER.md, note it in the relevant task's log entry.
diff --git a/.claude/skills/generate-report/SKILL.md b/.claude/skills/generate-report/SKILL.md
new file mode 100644
index 000000000..239a7aa76
--- /dev/null
+++ b/.claude/skills/generate-report/SKILL.md
@@ -0,0 +1,120 @@
+---
+name: generate-report
+description: Analyzes a promoted experiment directory and produces a structured markdown report under experiments/_reports/. Use after intaking and promoting HPC results to summarize findings. Also use when the user says "generate a report", "summarize results", or "write the experiment report".
+allowed-tools: Read, Write, Bash, Glob, Grep
+---
+
+# Generate Experiment Report
+
+## Step 1 -- Identify the experiment and task
+
+Ask (or infer from context): what is the promoted experiment path and which task does it belong to?
+
+```bash
+ls experiments/<task>/
+```
+
+The task name determines which log file to update: `logs/LOG.<task>.md`.
+
+## Step 2 -- Inventory results
+
+```bash
+# Check what's in the experiment root
+ls experiments/<task>/<experiment_name>/
+
+# Find all result/metric files
+find experiments/<task>/<experiment_name>/ -name "*.json" -o -name "*.csv" | head -30
+
+# Check existing reports
+ls experiments/<task>/<experiment_name>/_reports/ 2>/dev/null || echo "No reports yet"
+```
+
+## Step 3 -- Read key files
+
+- `status.json` -- final state and task metadata
+- `manifest.json` -- expected vs observed jobs
+- Per-run metric files (e.g. `results.json`, `metrics.csv`, `summary.json`)
+- Existing `README.md` inside the package
+
+## Step 4 -- Write the report
+
+Output path: `experiments/<task>/<experiment_name>/_reports/<report_name>/<report_name>.md`
+
+### Report structure
+
+```markdown
+# <Experiment Name> -- Report
+
+**Date:** YYYY-MM-DD
+**Task:** <task>
+**Status:** complete / incomplete (N/M runs)
+
+## Summary
+
+One paragraph: what was tested, key result, headline number.
+
+## Configuration
+
+Table of key hyperparameters swept.
+
+## Results
+
+### Main metric table
+
+| Method | Metric | Mean | P10 | Std |
+|--------|--------|------|-----|-----|
+
+### Figures
+
+Embed any generated plots (use relative paths).
+
+### Key findings
+
+Bullet list of the 3-5 most important observations.
+
+## Failure / Incomplete Jobs (if any)
+
+List missing jobs and their impact on conclusions.
+
+## Next Steps
+
+What should be done based on these results?
+```
+
+## Step 5 -- Update canonical summary
+
+Ensure `_reports/<report_name>/summary.json` exists with required fields:
+- `experiment_id`, `source_packages`, `expected_jobs`, `observed_jobs`, `missing_jobs`, `status`
+
+## Step 6 -- Bi-directional sync (required by AGENTS.md)
+
+### PAPER.md
+
+1. In the **Results** section: update the current best result for this task.
+2. If the previous best result is being displaced, move it (with its config) to **Appendix: Ablation History -- \<task\>**.
+3. In **Ablation Summary**: add or update the row for this variant in the per-task table.
+
+```markdown
+<!-- Ablation Summary table format -->
+| Run | Key change | Metric | vs. previous |
+|-----|------------|--------|--------------|
+```
+
+After adding the new row to the Ablation History table, re-sort it:
+
+```bash
+conda activate normal
+python scripts/sort_ablation_tables.py
+```
+
+### README.md
+
+Update the "Current Results" row for this task (one line per task, overwrite -- do not append).
+
+### logs/LOG.\<task\>.md
+
+Invoke `/log-entry` for this task. The entry should reference the report path and headline number.
+
+### LOG.md (global index)
+
+Update the row for `<task>` with the new Last Run, Status, and link.
diff --git a/.claude/skills/hpc-submit/SKILL.md b/.claude/skills/hpc-submit/SKILL.md
new file mode 100644
index 000000000..53a5b6362
--- /dev/null
+++ b/.claude/skills/hpc-submit/SKILL.md
@@ -0,0 +1,256 @@
+---
+name: hpc-submit
+description: Guides creation of a compliant SLURM submitter + runner script pair for this project. Use when preparing a new experiment for cluster submission, writing new HPC scripts, or auditing existing scripts against project policy. Also use when the user says "create a submitter", "write SLURM scripts", or "submit to cluster".
+allowed-tools: Read, Write, Edit, Bash, Glob, Grep
+---
+
+# HPC Submit — New Experiment
+
+Full policy: [docs/policy/HPC_RESULT_INTAKE_POLICY.md](docs/policy/HPC_RESULT_INTAKE_POLICY.md).
+
+**Cluster facts + resilience templates live in the global `pax-hpc` skill**
+(`~/.claude/skills/pax-hpc/`): cluster choice (NEW=login-p02/conda/2d cap;
+OLD=login-prod/Singularity-always/6d cap), VRAM-precise GPU labels
+(`references/new-cluster.md`), and the copy-paste requeue/resume/VRAM-guard
+scaffolding (`references/resilience.md`). This project skill is the *house style*
+on top of that. Verified on both clusters, re-confirmed 2026-07-17: `preempt`
+`GraceTime=0` (no window on a preempt kill — only the last periodic checkpoint
+survives; the 30 s `KillWait` grace is the *walltime*/`scancel` path, not
+preemption), `PreemptMode=REQUEUE` **does not auto-requeue** — so the resilience
+contract below is mandatory, not optional. **Preempt jobs also require
+`--qos=preempt`** (not just `-p preempt`): without it the job silently runs under
+the `normal` QOS (cpu=250) and gets none of the higher preempt ceiling. The
+`normal` and `preempt` GPU caps are separate 10-GPU pools, so preempt spillover
+buys up to 20 concurrent GPUs (verified 2026-07-17).
+
+## Script pair overview
+
+Every experiment needs exactly two scripts:
+
+| Script | Runs on | Does |
+|---|---|---|
+| `scripts/<task>/submit_<task>_<variant>.sh` | Gateway/login node | `sbatch` orchestration only |
+| `scripts/<task>/run_<task>_<variant>.sh` (or `.py`) | Compute node via SLURM | Training, logging, packaging |
+
+## Submitter checklist (`bash` on gateway)
+
+- [ ] `set -euo pipefail` at top
+- [ ] Verifies `sbatch` is available
+- [ ] Validates required files/paths before submission
+- [ ] Prints usage and key env overrides
+- [ ] GPU constraint default uses **VRAM-precise labels**, not bare type names:
+      `a100-80G|h100-80G|h200-141G|l40s-48G` (bare `a100` silently admits the
+      40 GB card → OOM). Use the `constraint_for_tier` recipe from the global
+      `pax-hpc` skill's `references/new-cluster.md`.
+- [ ] Supports override via `GPU_CONSTRAINT` or `SBATCH_CONSTRAINT` env vars
+- [ ] Constraint fallback: try full VRAM-precise alternation → retry token-by-token → retry without `--constraint`
+- [ ] **Resilience SBATCH directives on EVERY job** (walltime trap is universal —
+      see contract below): `--requeue`, `--signal=B:USR1@30`, `--open-mode=append`
+- [ ] **`preempt` jobs pass BOTH `--partition=preempt` and `--qos=preempt`** — the
+      QOS flag is required or you silently fall back to the `normal` ceiling (cpu=250)
+- [ ] `--time` within the cap of the target cluster (**2d new / 6d old**); never rely on the 15-min default
+- [ ] **No hardcoded `--exclude`** — bad nodes are learned at runtime by the runner
+- [ ] Token rotation: when splitting across tokens, rotate preference across jobs (not all on first accepted token)
+- [ ] One SLURM job per independent sweep point (mode × seed grid) — no bundling unless true dependency
+- [ ] Big / multi-stage work uses `--dependency` chains, not one mega-job
+- [ ] Does NOT run Python, training, plotting, or `singularity exec` on gateway
+
+## Runner checklist (compute node)
+
+- [ ] Writes only inside one unique package root per task (no out-of-package writes)
+- [ ] Emits all four required files inside package root:
+  - `status.json` with `task_id` and `state` (`finished`/`partial`/`failed`)
+  - `manifest.json` with `required_files` and `promote_to`
+  - `README.md` (human-readable run context)
+  - `INTAKE.md` (local post-transfer commands)
+- [ ] Uses fail-fast validation for required runtime inputs
+- [ ] Verifies required Python modules before training
+- [ ] Records bootstrap/failure outcome in package metadata
+- [ ] Logs node/hardware info at startup (see **Hardware & timing log block** below)
+- [ ] Logs job start time and wall-clock end time (success and failure paths)
+- [ ] Registers EXIT trap that logs reason + elapsed time on any exit (OOM, kill, error)
+- [ ] **VRAM guard**: read `nvidia-smi` at startup; if VRAM `< MIN_GPU_MEM_MB`,
+      append the node to `ExcNodeList` and `scontrol requeue` (don't just requeue —
+      exclude, or it loops onto the same small card)
+- [ ] **CUDA-init guard**: `torch.zeros(1, device="cuda")` probe AFTER env activate;
+      on failure, exclude-node + requeue (a healthy `nvidia-smi` can still have a dead CUDA ctx)
+- [ ] **Signal traps** (`SIGUSR1` + `SIGTERM`) on EVERY job: requeue on the walltime
+      warning (universal — a job that overruns `--time` must requeue, never die) AND on
+      preemption; on preemption requeue *first* (GraceTime=0 → no window). Gate the
+      requeue loop on a `DONE` sentinel so it stops once the work is complete. See contract below.
+- [ ] **Robust resume**: resolve the resume dir by probing for the **highest-step
+      checkpoint** across `run_*/`, not a single sentinel file; honor `RESUME_RUN_DIR`
+- [ ] **Deliverable checkpoints off by default** (see **Checkpoint policy** below):
+  - Training code does not call `torch.save` / `save_pretrained` / Keras `save_best=True`
+    for *deliverable weights* unless gated by `--save-checkpoint` (or `KEEP_CHECKPOINT=1`), default **off**
+  - `required_files` in manifest.json omits any `*.pt`/`*.pth`/`*.safetensors`/`*.bin` entries
+  - **EXCEPTION — resume checkpoints are mandatory for preempt/long jobs**: a single
+    latest, atomically-written (`*.tmp` → `mv`) checkpoint every ≤~30 min of compute,
+    **deleted on clean finish**. This is transient resume state, not a deliverable, so
+    it stays out of `required_files` but is required for the job to survive `GraceTime=0`.
+
+## Walltime policy (user directive 2026-07-11 — HARD RULE)
+
+- **`--time=23:55:00` is the MAXIMUM walltime for any job.** Do not request more.
+- **Every task MUST be able to requeue itself** on timeout OR on an HPC kill, and
+  resume from its last checkpoint — never from step 0. A job that cannot save and
+  resume cannot be submitted.
+- Rationale (measured): a tied-LoRA run at ~270M on FineWeb-Edu took **22h31m**;
+  a 14h walltime would have failed it. Long jobs are normal, so the 23:55 cap +
+  self-requeue is what makes them finish. Jobs longer than 23:55 complete by
+  requeueing across multiple windows, resuming from the periodic checkpoint.
+- This means the periodic atomic checkpoint (≤ ~30 min) is **mandatory for every
+  job**, not just preempt/near-cap ones — it is what a requeue resumes from.
+
+## Resilience contract (walltime trap MANDATORY for ALL jobs)
+
+> **User directive (2026-06-10): every HPC task traps the walltime SIGTERM and
+> auto-requeues until completed — no exceptions.** Walltime estimates are
+> routinely wrong; an overrun must resume, never silently die. The periodic
+> *checkpoint* layer below stays scoped to preempt / near-cap jobs (it's what
+> lets a requeue resume near where it died); the *trap + `--requeue`* layer is
+> universal. The requeue loop is gated on a `DONE` sentinel — requeue while
+> unfinished, stop once the run completes.
+
+
+Copy-paste bash/Python templates: global `pax-hpc` skill →
+`references/resilience.md`. The canonical in-repo example is
+[scripts/wikitext/run_wikitext_v6_slurm.sh](scripts/wikitext/run_wikitext_v6_slurm.sh)
+(already implements `--requeue`, USR1/TERM handlers, VRAM guard, cuda-exclude).
+
+Why mandatory: `preempt` has `GraceTime=0` on both clusters — a preempted job is
+SIGKILLed with **no checkpoint window**, and `PreemptMode=REQUEUE` does **not**
+auto-requeue unless the job is `--requeue`-eligible. Several tasks also exceed the
+cap (AWD-LSTM ≈3d vs the new 2d cap), so they *cannot finish* without resume.
+
+The contract has three layers — get all three or the job is not resilient:
+
+1. **Eligibility (submitter):** `#SBATCH --requeue` + `--signal=B:USR1@30` +
+   `--open-mode=append`, `--time` within the cluster cap. On `preempt`, also
+   `#SBATCH --qos=preempt` (required for the higher ceiling; a bare `-p preempt`
+   falls back to the `normal` QOS).
+2. **Survival (runner, bash):** trap `SIGUSR1`/`SIGTERM` → on preemption requeue
+   *first* then best-effort checkpoint, on walltime checkpoint-then-requeue;
+   VRAM-guard + cuda-guard that exclude-node + requeue; resume by highest-step probe.
+3. **State (trainer, Python):** trap `SIGUSR1` → write latest checkpoint → exit;
+   **periodic atomic checkpoint every ≤~30 min** (the only thing that survives a
+   0-grace preemption); resume from the highest-step checkpoint; delete on clean finish.
+
+Requeue lives in the **bash** layer (survives a hung trainer); the Python layer only
+checkpoints. The walltime trap + `--requeue` are NOT optional for any job —
+CPU-only and sub-hour jobs on `batch`/`gpu` still carry them (a wrong runtime
+estimate must requeue, not die); they may skip only the *periodic checkpoint*
+layer if a from-scratch restart on requeue is acceptable.
+
+## Checkpoint policy
+
+> Full policy: [docs/policy/HPC_RESULT_INTAKE_POLICY.md](docs/policy/HPC_RESULT_INTAKE_POLICY.md)
+> section **Checkpoint Persistence Policy**.
+
+**Default**: evaluation-only experiments persist config + metrics + history,
+not weights. `config + code commit + seed = same result`; checkpoints add no
+reproducibility and a lot of disk.
+
+**Opt in** (explicit flag, default off) only when:
+1. The weight artifact itself is the deliverable (released model, teacher, warmstart)
+2. Multi-stage training needs mid-run state (ReLoRA merge, warmstart handoff)
+3. Active investigation needs post-hoc weight probing (eigenvalues, activations)
+
+When opting in, record `extra.checkpoint_reason` in manifest.json and keep the
+file optional (never in `required_files`) so evaluation reruns don't trip
+`INCOMPLETE`.
+
+**Cleanup of legacy dirs**: `python scripts/strip_checkpoints.py --scan experiments/<task>/`
+removes `*.pt`/`*.pth`/`*.safetensors`/`*.bin` while preserving all metadata.
+
+## Hardware & timing log block
+
+Paste this block **once, near the top of every runner script**, immediately after the SLURM env debug section:
+
+```bash
+# =============================================================================
+# HARDWARE & TIMING INFO
+# =============================================================================
+JOB_START_TIME=$(date +"%Y-%m-%dT%H:%M:%S")
+JOB_START_EPOCH=${SECONDS}
+echo "===== JOB START: ${JOB_START_TIME} ====="
+echo "Node:       ${SLURMD_NODENAME:-unknown}"
+echo "NodeList:   ${SLURM_JOB_NODELIST:-unknown}"
+echo "JobID:      ${SLURM_JOB_ID:-unknown}"
+echo "ArrayTask:  ${SLURM_ARRAY_TASK_ID:-none}"
+echo "Cluster:    $(hostname -d 2>/dev/null || hostname)"
+echo "SLURM job: ${SLURM_JOB_ID}"
+
+# GPU info (model + VRAM)
+if command -v nvidia-smi &>/dev/null; then
+    echo "--- GPU ---"
+    nvidia-smi --query-gpu=index,name,memory.total,driver_version \
+        --format=csv,noheader,nounits 2>/dev/null \
+        | awk -F',' '{printf "  GPU %s: %s  |  VRAM: %s MiB  |  Driver: %s\n",$1,$2,$3,$4}'
+else
+    echo "GPU: not available"
+fi
+
+# CPU memory
+echo "--- CPU Memory ---"
+free -h | grep -E '^Mem'
+
+# Register EXIT trap: always log finish time + status + elapsed
+_log_job_exit() {
+    local exit_code=$?
+    local elapsed=$(( SECONDS - JOB_START_EPOCH ))
+    local end_time
+    end_time=$(date +"%Y-%m-%dT%H:%M:%S")
+    echo ""
+    echo "===== JOB EXIT: ${end_time} ====="
+    echo "Exit code:    ${exit_code}"
+    echo "Elapsed:      ${elapsed}s  (~$(( elapsed/3600 ))h $(( (elapsed%3600)/60 ))m $(( elapsed%60 ))s)"
+    if [ "${exit_code}" -eq 0 ]; then
+        echo "Status:       SUCCESS"
+    elif [ "${exit_code}" -eq 137 ]; then
+        echo "Status:       KILLED (OOM or external kill signal — check memory limits)"
+    else
+        echo "Status:       FAILED (see Python traceback above)"
+    fi
+    echo "Start:        ${JOB_START_TIME}"
+    echo "End:          ${end_time}"
+}
+trap '_log_job_exit' EXIT
+# =============================================================================
+```
+
+**Debug notes captured by this block:**
+- GPU model + VRAM → tells you if a different GPU type ran (A100 vs L40s affects numerics/OOM thresholds)
+- Exit code 137 → OOM kill (not a Python crash); increase `--mem` or reduce batch size
+- Elapsed time → compare against `--time` budget; if close, job was likely preempted or killed at limit
+- Python loss/parameter logs → emitted to the SLURM `.out` file by the training script; grep `loss` or `param` in that file to trace divergence
+
+## Policy alignment checklist
+
+- [ ] Config/wrapper paths remain under task-scoped roots (not repo root)
+- [ ] Script behavior aligns with `scripts/validate_experiment_policy.py`
+- [ ] Docs synchronized: `README.md`, `PAPER.md`, `LOG.md`
+- [ ] Aggregation of split-job outputs happens locally after intake
+
+## Common failure patterns to avoid
+
+1. `Invalid feature specification` — use bracket constraint with token fallback
+2. Running `bash submitter.sh` instead of `sbatch submitter.sh` (or vice versa)
+3. Python/container execution on gateway node
+4. Shared HF cache across parallel jobs — use per-job `HF_HOME` or `TRANSFORMERS_CACHE`
+5. Non-self-contained outputs (writes outside package root)
+6. **Silent OOM from a bare `a100` constraint** landing on a 40 GB card — use a
+   VRAM-precise label (`a100-80G`) + the runtime VRAM guard with `MIN_GPU_MEM_MB`
+7. **Preempted job vanishes instead of requeuing** — missing `#SBATCH --requeue`
+   (`PreemptMode=REQUEUE` alone does not requeue your job)
+8. **Resume starts from zero after ≥2 requeues** — trusting a stale sentinel file
+   instead of probing for the highest-step checkpoint
+
+## After writing scripts
+
+Run the policy validator:
+```bash
+python scripts/validate_experiment_policy.py
+```
+
+Then invoke `/log-entry` to record the new scripts.
diff --git a/.claude/skills/intake-results/SKILL.md b/.claude/skills/intake-results/SKILL.md
new file mode 100644
index 000000000..807ad891e
--- /dev/null
+++ b/.claude/skills/intake-results/SKILL.md
@@ -0,0 +1,101 @@
+---
+name: intake-results
+description: Runs the full HPC result intake pipeline for this project. Use when experiment results have been transferred from the cluster into experiments/from_HPC and need to be validated, promoted, and archived. Also use when the user says "intake", "promote results", or "process HPC output".
+allowed-tools: Bash, Read, Write, Edit, Glob, Grep
+---
+
+# Intake Results from HPC
+
+Full policy is in [docs/policy/HPC_RESULT_INTAKE_POLICY.md](docs/policy/HPC_RESULT_INTAKE_POLICY.md).
+
+## Step 1 -- Check what arrived
+
+```bash
+ls experiments/from_HPC/
+```
+
+If packages landed under a staging subdirectory (`hpc_staging`), normalize first:
+
+```bash
+bash scripts/hpc/intake_from_hpc_staging.sh --dry-run
+# If dry-run looks correct:
+bash scripts/hpc/intake_from_hpc_staging.sh --execute --run-intake --interactive
+```
+
+```bash
+# Extract SLURM job IDs from stdout logs in the incoming package
+grep -r "SLURM job:" experiments/from_HPC/ 2>/dev/null | head -20
+```
+
+Copy the job ID(s) into the log entry SLURM field when writing the entry.
+
+## Step 2 -- Run intake (dry-run first, always)
+
+```bash
+python scripts/hpc/hpc_result_intake.py --interactive --dry-run
+```
+
+Review the output, then run live:
+
+```bash
+python scripts/hpc/hpc_result_intake.py --interactive
+```
+
+## Step 3 -- Decide on each package
+
+| Classification | Condition | Action |
+|----------------|-----------|--------|
+| `COMPLETE` | `status.json` valid, `state=finished`, all required files present | Auto-promote |
+| `INCOMPLETE` | `state=partial/failed` or required files missing | Choose: rerun missing, rerun all, accept partial, or skip |
+| `UNKNOWN` | `status.json` missing or malformed | Do NOT silently promote -- investigate first |
+
+Quarantine path (if promotion fails): `experiments/_quarantine/<name>_<timestamp>/`
+Audit log: `experiments/from_HPC/intake_events.jsonl`
+
+## Step 4 -- Unify results
+
+```bash
+python scripts/hpc/unify_hpc_results.py --dry-run
+python scripts/hpc/unify_hpc_results.py --archive-complete
+```
+
+## Step 5 -- Identify the task
+
+From the promoted package, determine the task name. This is the subdirectory under `experiments/` where the package landed. All subsequent log writes target `logs/LOG.<task>.md`.
+
+## Step 6 -- Generate report
+
+After promotion, invoke `/generate-report` with the promoted experiment path. That skill handles PAPER.md, README.md, and the per-task log in one step.
+
+## Step 7 -- Post-intake refresh (single command)
+
+Run the wrapper. It writes `run.json` sidecars for the task's runs, archives
+`LOG.<task>.md` if it exceeds 10 active entries, regenerates STATUS, rebuilds
+`LOG.md`, and rebuilds the archive search index.
+
+```bash
+python scripts/post_intake_refresh.py --task <task>
+```
+
+For a multi-task intake pass the flag more than once, or omit `--task` to
+refresh every active task:
+
+```bash
+python scripts/post_intake_refresh.py --task wikitext --task surrogate
+python scripts/post_intake_refresh.py       # all tasks
+```
+
+This replaces the previous manual sequence of `write_run_sidecar.py` +
+`archive_log.py` + `generate_status.py` + `refresh_index.py` +
+`build_search_index.py`.
+
+## Step 8 -- Bi-directional sync (required by AGENTS.md)
+
+The `/generate-report` skill covers this. Confirm before closing the session:
+- [ ] `logs/LOG.<task>.md` has a new entry
+- [ ] `logs/STATUS.<task>.md` regenerated
+- [ ] `run.json` sidecars present for every promoted run
+- [ ] `LOG.md` global index row is updated
+- [ ] PAPER.md Results section is current
+- [ ] PAPER.md Ablation Summary row exists for this variant
+- [ ] README.md "Current Results" row is current
diff --git a/.claude/skills/log-entry/SKILL.md b/.claude/skills/log-entry/SKILL.md
new file mode 100644
index 000000000..7a7b8ef79
--- /dev/null
+++ b/.claude/skills/log-entry/SKILL.md
@@ -0,0 +1,205 @@
+---
+name: log-entry
+description: Appends a new structured entry to LOG.md following the project's experimental log format. Use after completing an experiment run, implementing a feature, fixing a bug, or any change that should be recorded. Also use when the user says "update the log", "add a log entry", or "record this run".
+allowed-tools: Read, Edit
+---
+
+# Add a Log Entry
+
+## Step 0 — Check and update the pinned block
+
+After writing the entry (Step 3), decide whether this run is a new best or
+second-best for this task:
+
+- Read the existing `## 📌 Current Best` block (if any)
+- Compare the new result against the current 🥇 and 🥈
+- If the new result is better than 🥇: demote current 🥇 to 🥈, set new 🥇
+- If the new result is better than 🥈 only: replace 🥈
+- If the new result is not top-2: leave the block unchanged
+- If no block exists yet: create it with this run as 🥇 (and 🥈 empty)
+
+**The pinned block must always appear before the first `## ` entry in the file.**
+When in doubt about ranking (e.g. the metric changed), leave the block as-is
+and note "metric changed — manual review needed" in the Notes row.
+
+### PAPER.md sync (required when pinned block changes)
+
+If the pinned block was updated (🥇 changed):
+
+1. Open `PAPER.md` and locate the Results section for this task.
+   It will be under `## Results` or a subsection like `### <task>`.
+2. Replace the current best result line with the new 🥇 values.
+3. Move the displaced result (the old 🥇) into the task's
+   `## Appendix: Ablation History — <task>` table as a new row.
+   Format: `| YYYY-MM-DD | Run NNN | <what changed> | <metric> | <value> |`
+4. Do not rewrite the whole Results section — surgical edit only.
+
+If the metric or task does not appear in PAPER.md Results yet, add it.
+If PAPER.md does not exist, skip this step and note it in the log entry.
+
+## Step 1 -- Identify the task
+
+Determine the task name from context (e.g. the experiment directory, user instruction,
+or the most recently modified code). The task name is the subdirectory under `experiments/`.
+
+The target file is `logs/LOG.<task>.md`.
+
+## Step 2 -- Read the current log
+
+```bash
+head -10 logs/LOG.<task>.md
+```
+
+Find the current run number from the first `## ` heading. New run number = last + 1.
+
+## Step 3 -- Write the entry
+
+Use today's date (YYYY-MM-DD). **Prepend** the new entry immediately after the file
+header and before the first existing `## ` heading -- logs are reverse-chronological
+(newest first).
+
+### Entry format
+
+```markdown
+## YYYY-MM-DD Run NNN (Short Title -- Key Distinction)
+
+```yaml meta
+run: NNN
+date: "YYYY-MM-DD"
+task: <task_name>
+type: experiment          # experiment | infrastructure | bugfix | intake
+slurm_job: <job_id>       # omit if not a SLURM run
+key_metric: <metric_name> # omit if no quantitative result
+key_value: <number>       # the single most important result
+key_unit: "%"             # omit if dimensionless
+walltime_mean_s: <number> # mean wallclock seconds across all tasks; omit if not a SLURM run
+walltime_std_s: <number>  # std of wallclock seconds across all tasks; omit if not a SLURM run
+params:
+  model: <model_name>
+  rank: <rank>
+  method: <method>
+  epochs: <N>
+  seeds: <N>
+tags: [<tag1>, <tag2>]
+```
+
+**SLURM job:** `<job_id>` | **Array:** `<array_range>` | **Cluster:** `<cluster_name>`
+
+### Objective
+
+One paragraph explaining what this run/change set out to do and why.
+
+### Differential Update
+
+**`path/to/changed/file.py`** (brief scope summary):
+- Bullet explaining each specific change and rationale
+
+**`configs/task/config.yaml`**:
+- What changed and why
+
+### Wallclock (required for SLURM runs)
+
+For any entry that corresponds to one or more SLURM jobs, always include a
+`### Wallclock` section in the entry body. Extract elapsed seconds from the
+`.out` files (the `Elapsed: <N>s` line written by the runner) and compute
+mean and std across all tasks in the sweep:
+
+```bash
+grep -h "Elapsed:" experiments/from_HPC/<task>/<prefix>.*.out \
+  | awk '{print $2}' | sed 's/s//' \
+  | python3 -c "
+import sys, statistics
+vals=[int(l) for l in sys.stdin]
+print(f'N={len(vals)}  mean={statistics.mean(vals):.0f}s  std={statistics.stdev(vals):.0f}s  min={min(vals)}s  max={max(vals)}s')
+"
+```
+
+Format the section as:
+
+```markdown
+### Wallclock
+
+**Overall (N=<N> tasks):** mean=<mean>s (<mean_h>h) ± <std>s (<std_h>h) | min=<min>s | max=<max>s
+
+| Config | N | mean (s) | std (s) | mean (h) |
+|---|---|---:|---:|---:|
+| <config> | <n> | <mean> | <std> | <h>h |
+```
+
+Include per-config breakdown whenever the sweep spans multiple configurations.
+Omit if all tasks share the same config (report overall only).
+
+### Smoke Test (if applicable)
+
+```
+result_1: shape OK
+result_2: value OK
+```
+
+Brief interpretation.
+
+---
+```
+
+### Metadata block rules
+
+The ` ```yaml meta ``` ` block is **required** for every entry:
+- Always include `run`, `date`, `task`, `type`
+- Include `key_metric`/`key_value`/`key_unit` when the entry reports a quantitative result
+- Include `slurm_job` when applicable
+- Include relevant hyperparameters in `params` (free-form dict, task-specific)
+- Add descriptive `tags` for filtering (e.g. `rank-sweep`, `promoted`, `negative-result`)
+
+The `**SLURM job:**` line is optional — omit it for non-SLURM runs (local tests, debug runs).
+When present it must be the first line of the entry body, before Objective.
+
+## Step 4 -- Check if archiving is needed
+
+```bash
+count=$(grep -c "^## " logs/LOG.<task>.md)
+echo "$count entries"
+```
+
+If count > 10, invoke `/archive-log` with `--task <task> --keep-last 5`.
+(Hot logs stay small so STATUS + grep remain fast.)
+
+## Step 5 -- Update the global index
+
+Open `LOG.md` and update the row for this task with the new Last Run and Status.
+
+## Step 6 -- Bi-directional sync (required)
+
+After writing the log entry, verify that PAPER.md, README.md, and code are
+consistent with what was logged. Any methodological change in code must be
+reflected in PAPER.md in the same session.
+
+## Step 7 — Never archive the pinned block
+
+The `## 📌 Current Best` block is permanent. Confirm it is still present at
+the top of the file after any archival operation:
+
+```bash
+head -5 logs/LOG.<task>.md | grep "📌"
+```
+
+If it is missing, restore it from the archive file's header or from memory.
+
+## Step 8 — Refresh STATUS, global index, and search index
+
+After every log entry, regenerate the STATUS file (what Claude loads by default),
+the global index, and the search index from ground truth:
+
+```bash
+conda activate normal
+python scripts/generate_status.py --task <task>
+python scripts/refresh_index.py
+python scripts/build_search_index.py
+```
+
+This replaces the manual row-update instruction. Do not hand-edit LOG.md or
+STATUS.<task>.md — both are regenerated from the per-task log.
+
+## Image embedding
+
+If embedding images, copy to `log/img/[original_name]_[YYYY-MM-DD-H_M_S].[ext]`
+first, then embed with a relative path.
diff --git a/.claude/skills/pax-hpc/SKILL.md b/.claude/skills/pax-hpc/SKILL.md
new file mode 100644
index 000000000..a3a95d06e
--- /dev/null
+++ b/.claude/skills/pax-hpc/SKILL.md
@@ -0,0 +1,292 @@
+---
+name: pax-hpc
+description: >-
+  Prepare and launch compute work on the user's two Tufts "Pax" SLURM HPC
+  clusters. Use this skill WHENEVER the user wants to run, batch, submit,
+  array, schedule, profile, or GPU-accelerate any job, experiment, script, or
+  training run that is meant for the cluster — even if they don't say the word
+  "cluster". Trigger on mentions of sbatch, srun, salloc, squeue, slurm,
+  "the cluster", "on pax", "submit a job", "run this on a GPU", singularity /
+  apptainer containers, conda environments meant for the cluster, login-p02,
+  login-prod-03, preempt / requeue / resume, or any request to turn local code
+  into something that runs on HPC. There are two clusters: OLD (login-prod-03,
+  RHEL7, runs programs via Singularity) and NEW (login-p02, Rocky9, runs
+  programs via conda). Always ask which one first.
+---
+
+# Pax HPC (two clusters)
+
+This skill captures how the user works with their two Tufts Pax SLURM clusters
+so they never have to re-explain it. Both clusters share ONE filesystem, so
+data, repos, conda envs, and Singularity images live in the same place on both.
+The clusters differ in OS, software runtime, walltime limits, and GPU hardware.
+
+| | OLD cluster | NEW cluster |
+|---|---|---|
+| Login host | `login-prod-0X.pax.tufts.edu` (01, 03, …) | `login-p02.pax.tufts.edu` |
+| Compute nodes | legacy `cc1gpu / s1cmp / p1cmp / d1cmp###` | `pax001`–`pax120` |
+| OS / Slurm | RHEL 7.5 / Slurm 23.02 | Rocky 9.6 / Slurm 23.11 |
+| Runs programs via | **Singularity** (`module load singularity`) | **conda** (`conda activate`) |
+| Max walltime | 6 days | **2 days** |
+| GPUs | p100, v100, t4, a100(40G), + some newer | h100, h200, a100(40/80G), l40s, + more |
+| Why not the other runtime | host glibc too old for modern conda binaries | conda is native; apptainer also available |
+
+Username: `hhazan01`. Full hardware, partitions, QOS, and GPU **VRAM-tagged
+feature labels** are in `references/old-cluster.md` and
+`references/new-cluster.md` — read the relevant one when sizing a job.
+Resilience mechanics (requeue, resume, preempt signal-trap, walltime
+self-requeue, adaptive node-exclude, VRAM guard, dependency chains) live in
+`references/resilience.md`. Bringing results back and classifying them lives in
+`references/intake.md`.
+
+## The non-negotiable rules
+
+1. **Never run ANY task on a login node — under any circumstances** (OLD or NEW).
+   This is absolute (user directive, 2026-06-07). No training, no data crunching,
+   no `python long_thing.py`, no compiling, no `singularity exec` of a workload —
+   **and also no "lightweight" inline work**: no `python scripts/intake.py`,
+   `consolidate.py`, results-aggregation, jsonl parsing, or env-python one-liners
+   on the login node, even for a quick verification. If you need to run code
+   against cluster data, submit it (`sbatch`) or use an interactive compute
+   allocation (`srun --pty`) — or pull the data local and run it here. The login
+   node is ONLY for: editing files, submitting jobs (`sbatch`), launching
+   allocations (`srun --pty`/`salloc`), monitoring (`squeue`, `sacct`,
+   `scancel`), small `ls`/`tail`/`grep` on output files, and `rsync`. **This
+   applies to `login-p02` even when it is used as a no-Duo ProxyJump gateway to
+   the OLD cluster — gateway convenience is not a compute license.** When in
+   doubt, it does not run on login: pull the data local and run it here, or
+   `sbatch`/`srun`.
+
+2. **Always ask which cluster first: OLD or NEW?** The answer changes the
+   runtime wrapper (Singularity vs conda), the walltime ceiling (6d vs 2d), and
+   which GPUs are valid. Do not guess. (If the user already said this session,
+   don't re-ask.)
+
+3. **Prepare everything locally.** Build the job script and any code/config in a
+   local staging folder. Do not assume you can write on the cluster.
+
+4. **The user does the rsync** unless they explicitly say otherwise. Give them
+   the exact `rsync` command, but they run it.
+
+5. **End every preparation with ONE line** the user pastes on the login node to
+   launch — normally `sbatch <script>` (or an array/dependency line). That single
+   line is the deliverable of local prep.
+
+6. **The preempt contract — EVERY `preempt` job carries it, no exceptions**
+   (user directive, 2026-06-20). A job on the `preempt` partition that is not
+   wired for kill→requeue→resume is a bug, not a job. The full contract:
+   - **`#SBATCH --requeue`** — makes the job requeue-eligible. Without it,
+     preemption *cancels* the job (it does not come back). Mandatory.
+   - **Trap the kill signals SLURM sends to stop the process** — the runner
+     traps **`SIGUSR1` + `SIGTERM` + `SIGCONT`** (request the advance warning
+     with `#SBATCH --signal=B:USR1@30`) and routes them to one handler that
+     `scontrol requeue "$SLURM_JOB_ID"`s **first**, then forwards a checkpoint
+     request to the trainer. Requeue first so a slow checkpoint can never cost
+     you the restart. Do the `scontrol requeue` in bash, the checkpoint in
+     Python (§0.1 of resilience.md).
+   - **Auto-requeue until done** — the requeued job restarts the batch script
+     from the top and keeps going until a completion (`DONE`) sentinel exists;
+     `SLURM_RESTART_COUNT` caps runaway loops.
+   - **Periodic checkpoint + resume-from-last is MANDATORY, not optional.**
+     `GraceTime=0` on `preempt` (both clusters, verified) means preemption gives
+     **no on-signal checkpoint window at all** — SIGTERM and SIGKILL arrive
+     back-to-back. The *only* thing that survives is the last **periodic
+     checkpoint** (write every K steps, ≤~30 min of compute at risk; write
+     atomically — tmp + `os.replace`). On restart the job MUST detect the
+     checkpoint and **resume from the last completed step**, never from step 0.
+     A preempt job whose code cannot save and resume from a checkpoint cannot
+     run on `preempt` — fix the resume path first or use a non-preempt partition.
+   - Verified facts to size against: `PreemptMode=REQUEUE`, `GraceTime=0`,
+     `KillWait=30`. Full copy-paste scaffolding in `references/resilience.md`
+     §0–§2.
+
+7. **EVERY task auto-requeues on walltime until it finishes — but ONLY on
+   walltime, and NEVER a crash, a cancel, or a completed job** (user directive,
+   2026-06-10/-12). Walltime estimates are routinely wrong; a job that overruns its
+   `--time` must resume, not die. Every runner carries `#SBATCH --requeue` +
+   `--signal=B:USR1@N`, **traps `SIGUSR1` + `SIGTERM` + `SIGCONT`**, forwards a
+   checkpoint request to the child, and after the child is reaped requeues
+   **iff: (a) the `SIGUSR1` walltime warning fired (`WALLTIME=1`) AND (b) no
+   completion sentinel (unfinished)**. The requeue is **rc-independent**: a walltime
+   stop may exit 0 (checkpointed in time) or non-zero (hard-killed) — `WALLTIME=1`
+   already excludes crashes and `scancel` (neither sends `SIGUSR1`), and the sentinel
+   excludes a clean finish. Trapping `TERM`/`CONT` only keeps the shell alive to make
+   that decision; a `scancel` (also `TERM`) carries `WALLTIME=0`, so it is never
+   requeued. `SLURM_RESTART_COUNT` caps the loop. Preempt-partition jobs differ (no
+   warning) — see §1b. Refs: `references/resilience.md` §0.2, §0.3, §1, §1b.
+
+## Standard workflow
+
+1. **Ask: OLD or NEW cluster?** Then read that cluster's reference file.
+2. **Clarify the job shape**: CPU or GPU, cores, memory, expected walltime,
+   single job vs array vs dependency chain, which conda env (new) or `.sif`
+   image (old). Size it against the reference file's limits.
+3. **Pick the GPU robustly** (see *GPU requests* below) — by VRAM constraint,
+   not by bare typed gres.
+4. **Wire resilience — always.** Every job gets the walltime SIGTERM trap +
+   `--requeue` + resume scaffold from `references/resilience.md` (rule 7).
+   `preempt` jobs and jobs near the cap additionally need the periodic
+   checkpoint (≤~30 min at risk) so they resume near where they died, not from
+   step 0.
+5. **Write the script(s) locally** in a staging dir (default `./<project>_job/`).
+6. **Give the rsync command** to the shared project space. Default destination:
+   `/cluster/tufts/levinlab/hhazan01/<project>/`.
+7. **Give the one launch line**, e.g.
+   `cd /cluster/tufts/levinlab/hhazan01/<project> && sbatch run.sbatch`.
+8. **Tell them how to pull results back** (`references/intake.md`).
+
+## Paths (shared across BOTH clusters)
+
+- **Home:** `/cluster/home/hhazan01` — **small (~28 GB quota, usually near
+  full)**. Do NOT stage data, datasets, package caches, or run output here. Use
+  it only for dotfiles and small scripts.
+- **Lab / project space:** `/cluster/tufts/levinlab/hhazan01` — main working
+  area. Conda envs, Singularity images, git repos, datasets, and staged jobs all
+  live here. `~/cluster` already symlinks to it.
+- **Scratch:** `/cluster/scratch` — large, fast, for transient run output.
+- **Conda envs (new):** `/cluster/tufts/levinlab/hhazan01/miniconda3/envs/*`
+  (verified 2026-05-31: `betse`, `bindsnet`, `delayW`, `dt-d4rl`, `LoRa`).
+- **Singularity images (old):** `/cluster/tufts/levinlab/hhazan01/singularity/*.sif`
+- `umask 0007` is already in `.bashrc` (group-shared lab files).
+
+Because home is tiny, always point conda package dirs, HuggingFace caches, torch
+extensions, and dataset downloads into the lab/scratch space, not home. Do not
+write large checkpoints unless the weight artifact is itself the deliverable —
+config + code commit + seed reproduces a result; checkpoints just burn disk.
+(Checkpoints written *for resume* are exempt — keep only the latest, delete on
+clean finish.)
+
+## Job script templates
+
+Both clusters use the same SBATCH header style; only the run wrapper differs.
+Keep `--time` within the ceiling (6d old, **2d new**).
+
+### Shared SBATCH header
+
+```bash
+#!/usr/bin/env bash
+#SBATCH --job-name=NAME
+#SBATCH --output=./res_%j_%N.txt
+#SBATCH --error=./res_%j_%N.err
+#SBATCH --time=00-04:00:00          # DD-HH:MM:SS  (<=2d new, <=6d old)
+#SBATCH --partition=PARTITION       # see reference file
+#SBATCH --nodes=1
+#SBATCH --ntasks=1
+#SBATCH --cpus-per-task=4
+#SBATCH --mem=16G
+## GPU jobs: see "GPU requests" below — prefer --gres=gpu:1 + --constraint=<VRAM label>
+## Arrays add, e.g.:    #SBATCH --array=1-20   (MaxArraySize=2000)
+## Resilient jobs add:  #SBATCH --signal=B:USR1@30  --open-mode=append  --requeue
+```
+
+### NEW cluster (login-p02) — conda
+
+```bash
+source ~/.bashrc
+conda activate ENV_NAME          # e.g. LoRa, delayW, betse, bindsnet, dt-d4rl
+cd /cluster/tufts/levinlab/hhazan01/PROJECT
+python my_script.py --args ...
+```
+
+### OLD cluster (login-prod-03) — Singularity
+
+Conda will NOT run here (host glibc too old). Wrap the workload in the container:
+
+```bash
+module load singularity
+SIF=/cluster/tufts/levinlab/hhazan01/singularity/IMAGE.sif
+cd /cluster/tufts/levinlab/hhazan01/PROJECT
+singularity exec --nv --bind /cluster:/cluster "$SIF" \
+    python3 my_script.py --args ...
+```
+
+`--nv` enables GPUs (drop it for CPU-only). `--bind /cluster:/cluster` makes the
+shared filesystem visible inside the container. `--cleanenv` helps when host env
+vars leak in.
+
+## GPU requests (the #1 source of silent OOMs)
+
+**Typed gres does not pin VRAM.** `--gres=gpu:a100:1` can land you on a 40 GB
+A100 *or* an 80 GB A100 — they share the type name `a100`. A 300M+ model that
+needs 80 GB will then OOM at runtime with no obvious cause. **Request by VRAM
+constraint instead:**
+
+```bash
+#SBATCH --gres=gpu:1
+#SBATCH --constraint=a100-80G        # VRAM-precise; see reference file for labels
+```
+
+- Use the VRAM-tagged feature labels (`a100-40G`, `a100-80G`, `h200-141G`,
+  `l40s-48G`, `h100-80G`, …) from the cluster reference file. For a "≥80 GB"
+  job use the alternation `a100-80G|h100-80G|h200-141G`, never coarse
+  `a100|h100|h200` (which silently admits 40 GB cards).
+- **Belt-and-suspenders:** even with a constraint, every A100 (or mixed-VRAM)
+  request must carry the runtime **VRAM guard** from `references/resilience.md`:
+  the runner reads `nvidia-smi` at startup and, if it landed below
+  `MIN_GPU_MEM_MB`, excludes that node and requeues automatically. Constraint +
+  guard together make a wrong-card landing self-correcting.
+- **Constraint fallback ladder** (avoids `Invalid feature specification` hard
+  failures): submit with the full alternation → on reject, retry token-by-token
+  → last resort, retry with no constraint. See `references/resilience.md`.
+
+## Excludes are learned, not hardcoded
+
+**Every submission starts with NO `--exclude`.** Do not bake a stale bad-node
+list into submitters. Instead, the runner discovers a bad node at runtime (wrong
+VRAM, `libtorch_global_deps.so` NFS error, CUDA init failure) and requeues
+itself with that node added to a job-scoped `ExcNodeList`. This way each fresh
+submission re-tests previously-bad nodes (they get fixed), and a still-broken
+node is walked away from within seconds. Mechanism in `references/resilience.md`.
+
+## Splitting big tasks — dependency chains
+
+For work that exceeds one allocation (a multi-stage pipeline, or a run longer
+than the 2d/6d cap split into segments), chain jobs with SLURM dependencies
+rather than one mega-job:
+
+```bash
+jid1=$(sbatch --parsable stage1.sbatch)
+jid2=$(sbatch --parsable --dependency=afterok:$jid1 stage2.sbatch)
+jid3=$(sbatch --parsable --dependency=afterok:$jid2 stage3.sbatch)
+```
+
+`afterok` = run only if the prior succeeded; `afterany` = run regardless (use
+for a resume segment that should start even if the prior hit the walltime);
+`singleton` = serialize same-name jobs. Details and resume-segment patterns in
+`references/resilience.md`.
+
+## Interactive sessions (debugging only, still NOT the login node)
+
+`srun --pty` grabs a compute allocation for quick interactive testing — that is
+a compute node, not the login node. Prefer batch for anything real. Example:
+`srun -p preempt --time=2:00:00 -c 4 --mem=8g --pty bash`.
+
+## Queue ceiling — fine to over-submit
+
+`QOSMaxGRESPerUser` caps *simultaneously running* GPU jobs (~10). Submitting more
+than that is fine — the extras simply sit `PENDING` with reason
+`QOSMaxGRESPerUser` and start as running ones finish. Queue depth is not a
+problem; just submit the whole sweep.
+
+## Preflight checklist before handing over the launch line
+
+- Chosen cluster confirmed (old/new) and matching runtime wrapper used.
+- `--time` within ceiling (2d new / 6d old); default partition time is only
+  15 min, so always set `--time`.
+- Partition exists on that cluster and matches CPU/GPU need.
+- GPU jobs: `--gres=gpu:1` + a **VRAM `--constraint`** (not typed gres) + the
+  runtime VRAM guard for any A100/mixed-VRAM request.
+- No hardcoded `--exclude`; bad nodes are learned at runtime.
+- **Every job**: walltime SIGTERM trap + `--signal=B:USR1@30` + `--requeue` +
+  `--open-mode=append`, auto-requeuing until a `DONE` sentinel (rule 7).
+- **`preempt` jobs MUST carry the full preempt contract (rule 6)**: `--requeue`
+  + `SIGUSR1`/`SIGTERM`/`SIGCONT` trap → `scontrol requeue` + periodic atomic
+  checkpoint + resume-from-last-step. No preempt job without a working
+  save/resume path. Near-cap jobs need the same periodic checkpoint
+  (`references/resilience.md`).
+- `--mem` / `--cpus-per-task` fit a real node in the reference file.
+- Output/data paths point at lab or scratch space, not home.
+- Big/multi-stage work uses dependency chains, not one mega-job.
+- Final answer ends with the single `sbatch` (or dependency/array) line to run,
+  and a note on how to pull results back (`references/intake.md`).
+```
diff --git a/.claude/skills/pax-hpc/references/intake.md b/.claude/skills/pax-hpc/references/intake.md
new file mode 100644
index 000000000..fd741a607
--- /dev/null
+++ b/.claude/skills/pax-hpc/references/intake.md
@@ -0,0 +1,92 @@
+# Closing the loop — pulling results back and classifying them
+
+The skill's main job ends at a launch line, but a run isn't done until its
+results are back on the user's machine, validated, and filed. This is the return
+half. It has two parts: (1) jobs write **self-describing packages** on the
+cluster, and (2) after the user rsyncs them back, you **classify and
+disseminate** them.
+
+The user runs the rsync. You give the command and do the classification.
+
+---
+
+## 1. On the cluster: write a self-describing package per run
+
+A bare directory of `.csv`/`.pt` files is not intake-able — you can't tell a
+finished run from one the walltime killed. Each run's output dir should contain:
+
+- `status.json` — the single source of truth for "did this finish":
+  ```json
+  {"run_id": "myrun_s42", "state": "finished",
+   "exit_code": 0, "started": "...", "ended": "...",
+   "config": {"model": "...", "seed": 42, "lr": 3e-4},
+   "metrics": {"test_ppl": 25.82, "val_ppl": 25.1},
+   "git_commit": "abc1234"}
+  ```
+  `state ∈ {finished, partial, failed}`. Write `partial` from the resilience
+  trap when requeuing, `finished` only on clean completion, `failed` from the
+  EXIT trap on error.
+- `manifest.json` — `{"required_files": ["results.json","history.csv"], "promote_to": "<task>/<variant>"}`.
+  List only what a downstream reader needs; **do not** put resume checkpoints in
+  `required_files` (they're deleted on clean finish).
+- `README.md` — human context: what this run was, how to reproduce
+  (config + commit + seed), node/GPU it ran on.
+
+Have the EXIT trap stamp `status.json` on *every* exit path (success, error,
+OOM-137, requeue) so a killed run is still self-describing.
+
+---
+
+## 2. The user rsyncs results back
+
+Pull only the small metadata/metrics, not multi-GB checkpoints:
+
+```bash
+# from the user's local machine — they run this
+rsync -avz --prune-empty-dirs \
+  --include='*/' \
+  --include='status.json' --include='manifest.json' \
+  --include='README.md'   --include='results.json' --include='*.csv' \
+  --exclude='*' \
+  hhazan01@login-p02.pax.tufts.edu:/cluster/tufts/levinlab/hhazan01/PROJECT/out/ \
+  ./from_hpc/PROJECT/
+```
+
+(Use `login-prod-03` for the old cluster.) The `--include/--exclude` filter
+keeps weights on the cluster. If a checkpoint genuinely *is* the deliverable,
+add `--include='*.pt'` deliberately.
+
+---
+
+## 3. Classify and disseminate (you do this)
+
+Walk each package under `./from_hpc/` and sort it by `status.json`:
+
+| Classification | Condition | Action |
+|---|---|---|
+| **COMPLETE** | `status.json` valid, `state=finished`, all `required_files` present | Promote to its `promote_to` destination |
+| **INCOMPLETE** | `state=partial`/`failed`, or a required file missing | Offer: rerun missing, rerun all, accept partial, or skip — never silently promote |
+| **UNKNOWN** | `status.json` missing or unparseable | Investigate first; quarantine, do not promote |
+
+- **Promote**: move the package to its final task/variant location; record one
+  log line (what ran, key metric, commit, seed, node/GPU).
+- **Quarantine**: move UNKNOWN/failed-but-kept packages to a
+  `_quarantine/<name>_<timestamp>/` dir so they don't pollute the promoted tree
+  but aren't lost.
+- **Dedup requeued runs**: a resilient job can leave several `run_*` dirs from
+  successive requeues. Keep the one with the highest step / `state=finished`;
+  quarantine the rest. Classify by `(run_id, seed)`, not by directory count, or
+  you double-count requeues.
+- **Audit trail**: append one line per package to an `intake_events.jsonl`
+  (`{ts, package, classification, action, dest}`) so a re-run of intake is
+  idempotent and you can answer "did everything land?" without re-walking.
+
+---
+
+## 4. If this is a project with its own intake pipeline
+
+Some projects ship a dedicated intake (validator + promoter + log/STATUS
+regeneration). **Defer to the project's pipeline when one exists** — this file is
+the generic fallback for projects that don't. The contract is the same either
+way: self-describing packages on the cluster → rsync the metadata back →
+classify by `status.json` → promote / quarantine → record what landed.
diff --git a/.claude/skills/pax-hpc/references/new-cluster.md b/.claude/skills/pax-hpc/references/new-cluster.md
new file mode 100644
index 000000000..3cdad87d3
--- /dev/null
+++ b/.claude/skills/pax-hpc/references/new-cluster.md
@@ -0,0 +1,125 @@
+# NEW cluster — login-p02.pax.tufts.edu
+
+Modern cluster. Use **conda** to run programs. Apptainer/Singularity 1.4.2 is
+also available at `/usr/bin` if you need a container.
+
+- OS: Rocky Linux 9.6 · Slurm 23.11.11
+- Login: `login-p02.pax.tufts.edu` (the OLD cluster is `login-prod-03`)
+- Compute nodes are named `pax001`–`pax120` (growing; verify the top end with
+  `sinfo`).
+- Runtime: `source ~/.bashrc && conda activate <env>`
+- Envs live in `/cluster/tufts/levinlab/hhazan01/miniconda3/envs/`
+  (verified 2026-05-31): `betse`, `bindsnet`, `delayW`, `dt-d4rl`, `LoRa`.
+  Confirm current list with `conda env list`.
+
+## Scheduler limits (config)
+
+- Max walltime: **2 days** (`2-00:00:00`). Default time if unset: only 15 min —
+  always set `--time`. For runs longer than 2d, use in-place requeue+resume or a
+  dependency chain (see `resilience.md`).
+- Default memory per CPU: 2000 MB. `SelectType=cons_tres`, `CR_CPU_MEMORY`.
+- Max array size: 2000. Max job count: 100000.
+- Your QOS: `interactive`, `normal` (default).
+
+## Partitions
+
+| Partition | Use | Priority | Notes |
+|---|---|---|---|
+| `batch` | default CPU jobs | normal | QoS `normal` (cpu≤250, gpu≤10, mem≤5000G) |
+| `gpu` | lab-allocated GPU jobs | normal | reserved GPU nodes (h100/h200/a100/l40s) — preferred for training |
+| `preempt` | opportunistic, largest pool | low, **requeued when preempted** | QoS `preempt` (cpu≤1000, gpu≤10, mem≤10000G); most nodes incl. many GPUs. **Jobs here MUST be requeue-safe — see `resilience.md`.** |
+
+QOS ceilings: `normal` 2d / cpu=250 / gpu=10 / mem=5000G; `preempt` 2d /
+cpu=1000 / gpu=10 / mem=10000G; `normal-7days` 7d but **no GPU**; `interactive`
+4h / cpu=16 / gpu=1 / mem=64G. `QOSMaxGRESPerUser` caps simultaneously-*running*
+GPU jobs (~10) — over-submitting is fine, extras just queue.
+
+## CPU node types (cores / memory)
+
+- sapphirerapids: 64 cores, ~510 GB (the bulk of `batch`/`preempt`)
+- broadwell: 36–40 cores, 128–257 GB; some 515 GB and 1 TB large-mem
+- Constrain arch with `--constraint=sapphirerapids` or `broadwell` if needed.
+
+## GPUs — request by VRAM label, not bare type
+
+**Do not use typed gres (`--gres=gpu:a100:1`) for memory-sensitive jobs** — the
+type `a100` exists at both 40 GB and 80 GB, so a typed request can land on a
+40 GB card and OOM a 300M+ model. Request generic gres + a **VRAM-precise
+constraint**:
+
+```bash
+#SBATCH --gres=gpu:1
+#SBATCH --constraint=a100-80G       # or an alternation; see tiers below
+```
+
+VRAM-tagged feature labels, GRES type, and where they live (verified
+`sinfo` 2026-05-31):
+
+| GPU | VRAM | Feature label | GRES type | Partition (nodes) |
+|---|---|---|---|---|
+| H200 | 141 GB | `h200-141G` | `h200` | gpu: pax008–011 ·8; preempt: pax047 ·2 (lulab) |
+| H100 | 80 GB | `h100-80G` | `h100` | gpu: pax063 ·3 |
+| A100 | 80 GB | `a100-80G` | `a100` | gpu: pax007, pax049–050, pax105–106 ·8 (icelake) |
+| A100 | 40 GB | `a100-40G` | `a100` | gpu: pax003 ·2, pax051–052 ·8 (cascadelake) |
+| L40S | 48 GB | `l40s-48G` | `l40s` | gpu: pax020–026 ·4; preempt: pax064 ·4 (laolab) |
+| L40 | 48 GB | `l40-48G` | `l40` | preempt: pax048 ·4 (pettilab), pax110–113 ·4 (linlab) |
+| RTX 6000 Ada | 48 GB | `rtx_6000_ada-48G` | `rtx_6000_ada` | preempt: pax062 ·4 (heldweinlab) |
+| RTX A6000 | 48 GB | `rtx_a6000-48G` | `rtx_a6000` | preempt: pax119 ·8 (hugheslab) |
+| RTX A5000 | 24 GB | `rtx_a5000-24G` | `rtx_a5000` | preempt: pax065–068 ·8 (dinglab) |
+| RTX 6000 | 24 GB | `rtx_6000-24G` | `rtx_6000` | preempt: pax118 ·8 (hugheslab) |
+| T4 | 16 GB | `t4-16G` | `t4` | preempt: pax077, pax104 ·4 (linlab) |
+
+- **The type `a100` spans both 40 GB and 80 GB** (different nodes) — this is the
+  concrete reason `--gres=gpu:a100:1` can OOM. Pin with `--constraint=a100-80G`.
+- The `gpu` partition holds the lab/RT-allocated cards (feature `rtgpu`);
+  everything else is lab-private and surfaces only on `preempt` (so it's
+  backfill, and must be requeue-safe — `resilience.md`).
+- Node↔GPU assignments drift and the node count grows; the **labels** are durable,
+  the node names are not. **Never hardcode node names** — query live with
+  `sinfo -N -p gpu,preempt -o '%N %f %G'`.
+- The libtorch-NFS breakage that once hit pax009/010/011/024/025/026/063 is
+  cleared as of 2026-05-31 (all healthy with GRES) — exactly why excludes are
+  learned at runtime (`resilience.md` §4), never baked in.
+
+### Tier helper (drop into a submitter)
+
+```bash
+constraint_for_tier() {
+    case "$1" in
+        flex)      echo "" ;;                                                  # any GPU
+        ge_24g)    echo "rtx_a5000-24G|l40-48G|l40s-48G|rtx_6000_ada-48G|a100-40G|a100-80G|h100-80G|h200-141G" ;;
+        ge_40g)    echo "a100-40G|a100-80G|h100-80G|h200-141G|l40-48G|l40s-48G|rtx_6000_ada-48G" ;;
+        ge_48g)    echo "l40-48G|l40s-48G|rtx_6000_ada-48G|a100-80G|h100-80G|h200-141G" ;;
+        ge_80g)    echo "a100-80G|h100-80G|h200-141G" ;;                       # ≥80 GB, no 40G slip
+        a100_80)   echo "a100-80G" ;;                                          # avoid H100/H200 contention
+        h200)      echo "h200-141G" ;;
+        h100_h200) echo "h100-80G|h200-141G" ;;
+        *)         echo "" ;;
+    esac
+}
+```
+
+Tier guidance:
+- 300M body, dense or LoRA r≥256 → **`ge_80g`** (the 40 GB card OOMs).
+- 600M+ FFN-trainable / dense ablations → **`h200`** (80 GB cards peaked ~76 GB
+  and OOMed in past runs).
+- 2B body → **`h200`**, no exceptions.
+- Tiny pilots (≤10M) → **`flex`**, or `ge_24g` if the runner needs ≥24 GB.
+
+Always pair the constraint with the runtime **VRAM guard** (`resilience.md` §3)
+and set `MIN_GPU_MEM_MB` to match — the guard requeues off a wrong-VRAM landing
+instead of silently OOMing.
+
+Example GPU header:
+
+```bash
+#SBATCH --partition=gpu
+#SBATCH --gres=gpu:1
+#SBATCH --constraint=a100-80G
+#SBATCH --cpus-per-task=8
+#SBATCH --mem=64G
+#SBATCH --time=01-00:00:00
+```
+
+> The user's old `.bashrc` GPU aliases reference `p100`/`k20xm` — those are
+> OLD-cluster GPUs and do not exist here. Pick from the table above.
diff --git a/.claude/skills/pax-hpc/references/old-cluster.md b/.claude/skills/pax-hpc/references/old-cluster.md
new file mode 100644
index 000000000..121abfb3a
--- /dev/null
+++ b/.claude/skills/pax-hpc/references/old-cluster.md
@@ -0,0 +1,105 @@
+# OLD cluster — login-prod-03.pax.tufts.edu
+
+Legacy cluster. **Use Singularity to run programs — always, by user policy.**
+conda fails on the cluster's `rhel7` nodes (glibc < 2.28 → `GLIBC_2.28 not
+found`), which are the majority of the pool and **include every A100**. The
+`rhel8` nodes (l40 / h100 / l40s / rtx_a5000, on `preempt`) *could* in principle
+run conda via `-C rhel8`, but that would (a) lose all A100 access and (b) split
+the pool into two runtimes. **The user deliberately uses Singularity for ALL
+old-cluster jobs to avoid that complication — do not propose conda here.**
+Containerize the workload.
+
+- OS: RHEL 7.5 (Maipo) · Slurm 23.02.7
+- Login: `login-prod-0X.pax.tufts.edu` (there are several — e.g. `login-prod-01`,
+  `login-prod-03`).
+- Compute nodes keep the **legacy** naming `cc1gpu###`, `s1cmp###`, `p1cmp###`,
+  `d1cmp###` — this cluster did **not** migrate to `pax###` (only the NEW cluster
+  did). Don't expect `pax` names here.
+- Runtime: `module load singularity` then `singularity exec ...`
+  (apptainer/singularity is a module, not on the default PATH)
+- Images live at `/cluster/tufts/levinlab/hhazan01/singularity/*.sif`
+  (e.g. `mettagrid.sif`, `delayW.sif`, `LoRa.sif`, `Heb.sif` — confirm with `ls`)
+
+## Run wrapper
+
+```bash
+module load singularity
+SIF=/cluster/tufts/levinlab/hhazan01/singularity/IMAGE.sif
+singularity exec --nv --bind /cluster:/cluster "$SIF" python3 script.py --args
+```
+
+- `--nv` = expose GPUs (omit for CPU-only jobs)
+- `--bind /cluster:/cluster` = make the shared filesystem visible in-container
+- `--cleanenv` is useful when host env vars leak into the container
+
+## Scheduler limits (config)
+
+- Max walltime: **6 days** (`6-00:00:00`) — longer than the new cluster.
+- Default memory per CPU: 2000 MB. `cons_tres`, `CR_CPU_MEMORY`.
+- Max array size: 2000. Max job count: 100000.
+
+## Partitions
+
+| Partition | Use | Walltime | Notes |
+|---|---|---|---|
+| `batch` | default CPU jobs | 6d | rhel7/rhel8 nodes |
+| `gpu` | GPU jobs | 6d | a100/p100 reserved GPU nodes |
+| `mpi` | multi-node MPI | 6d | large core counts |
+| `largemem` | big-memory jobs | 6d | ~1 TB nodes |
+| `interactive` | quick interactive | 4h | for `srun --pty` |
+| `preempt` | opportunistic, largest/most varied pool | 6d | **requeued when preempted**; widest GPU selection. **Jobs here MUST be requeue-safe — see `resilience.md`.** |
+
+QOS ceilings: `preempt` cpu≤4000 / gpu≤20 / mem≤10000G; `public` cpu≤2000 /
+gpu≤10 / mem≤5000G; `expanded` cpu≤2000 / gpu≤24; `normal` default.
+
+## CPU node types
+
+- rhel8 nodes: 128 cores, ~510 GB (newest of the old pool)
+- broadwell rhel7: 36+ cores, 120–248 GB; large-mem up to ~1 TB / 1.5 TB
+- Constrain OS generation if a container needs it: `--constraint=rhel8` or
+  `--constraint=rhel7`.
+
+## GPUs
+
+Request generic gres + a VRAM/type constraint (same rationale as the new
+cluster — typed `--gres=gpu:a100:1` does not pin VRAM):
+
+```bash
+#SBATCH --partition=gpu        # or preempt
+#SBATCH --gres=gpu:1
+#SBATCH --constraint=a100      # tighten to a VRAM label if exposed (sinfo -o '%N %f %G')
+```
+
+Verified `sinfo` 2026-05-31. **VRAM feature labels exist only for A100 here** —
+everything else must be requested by GRES type, and a few labels are inconsistent
+(noted below):
+
+| GPU | VRAM | Feature label | GRES type | Nodes (count) |
+|---|---|---|---|---|
+| A100 | 40 GB | `a100-40G` ✓ | `a100` | cc1gpu001–005 ·8; p1cmp110–111 ·2 |
+| A100 | 80 GB | `a100-80G` ✓ | `a100` | s1cmp001–005 ·8; s1cmp006–007 ·2 |
+| H100 | 80 GB | **none** ⚠️ | `h100` | s1cmp010 ·3 — **no feature label; request by GRES only** |
+| RTX A6000 | 48 GB | `rtx_a6000` | `rtx_a6000` | s1cmp008 ·8 |
+| RTX 6000 Ada | 48 GB | `rtx_a6000ada` ⚠️ | `rtx_6000ada` ⚠️ | s1cmp009 ·4 — **feature/GRES names disagree** |
+| RTX A5000 | 24 GB | `rtx_a5000` | `rtx_a5000` | s1cmp012–015 ·8 |
+| L40 | 48 GB | `l40` | `l40` | d1cmp042–050 ·4 |
+| L40S | 48 GB | `l40` ⚠️ | `l40s` | s1cmp011 ·4 — **feature says `l40` but GRES is `l40s`** |
+| RTX 6000 | — | `rtx_6000` | `rtx_6000` | p1cmp077 ·2 |
+| V100 | — | `v100` | `v100` | p1cmp071–076 (·2–4) |
+| P100 | — | `p100` | `p100` | p1cmp073 ·4, p1cmp075 ·6 |
+| T4 | 16 GB | `t4` (no VRAM tag) | `t4` | p1cmp090–109 ·4 (≈20 nodes) |
+
+Practical rules for the old cluster:
+- **A100**: use `--constraint=a100-80G` / `a100-40G` (labels are reliable). Pair
+  with the runtime VRAM guard (`resilience.md` §3).
+- **H100**: there is no feature label — request `--gres=gpu:h100:1` directly.
+- **L40S vs L40**: don't trust the `l40` feature to mean 48 GB L40 specifically;
+  if you need L40S, request `--gres=gpu:l40s:1`. If you need L40, `--gres=gpu:l40:1`.
+- **RTX 6000 Ada**: feature is `rtx_a6000ada` but GRES is `rtx_6000ada` — use the
+  **GRES** form `--gres=gpu:rtx_6000ada:1`.
+- Always confirm live: `sinfo -N -p gpu,preempt -o '%N %f %G'`. Node names and
+  labels here are messier than the new cluster — never hardcode them.
+
+The user's `.bashrc` interactive GPU aliases (`sbash_gpu` → `gpu:p100:1`,
+`sbash_gpu_k20_x` → `gpu:k20xm:1` on partition `gpu`) were written for this
+cluster.
diff --git a/.claude/skills/pax-hpc/references/resilience.md b/.claude/skills/pax-hpc/references/resilience.md
new file mode 100644
index 000000000..00c8f2af2
--- /dev/null
+++ b/.claude/skills/pax-hpc/references/resilience.md
@@ -0,0 +1,457 @@
+# Resilience — requeue, resume, preemption, VRAM guard, dependencies
+
+Any job that can be interrupted must survive the interruption: catch the kill,
+save a checkpoint, requeue itself, and resume from that checkpoint on restart.
+
+**The walltime trap is universal (user directive, 2026-06-10): EVERY job — not
+just preempt or near-cap jobs — traps the SIGTERM the scheduler sends at the
+`--time` limit and `scontrol requeue`s itself, auto-requeuing until the run
+writes a `DONE` sentinel.** Walltime estimates are routinely wrong; a job that
+overruns must resume, never silently die. The *periodic checkpoint* layer
+(§0.2) stays scoped to **every `preempt` job** and **every job whose walltime
+approaches the cluster cap** (2 days new / 6 days old) — that's what lets a
+requeued job pick up near where it died instead of from step 0.
+
+This file gives copy-paste scaffolding. Drop the bash blocks into the runner
+(the script that executes on the compute node), not the submitter.
+
+---
+
+## 0. The signal facts (read first)
+
+Verified on the NEW cluster 2026-05-31: `KillWait=30 sec`,
+`PreemptMode=REQUEUE`, `PreemptType=preempt/partition_prio`,
+`PreemptExemptTime=0`.
+
+- **Signal 9 (SIGKILL) cannot be caught, trapped, or ignored.** Anything that
+  says "catch kill -9" is impossible. You catch the signal SLURM sends *before*
+  the SIGKILL, during a grace window.
+- On **preemption** (`preempt` partition), SLURM sends `SIGCONT` + `SIGTERM`,
+  waits the partition `GraceTime`, then `SIGKILL`. **`GraceTime=0` on the preempt
+  partition of BOTH clusters (verified 2026-05-31)** — so on preemption there is
+  **no on-signal window at all**; SIGTERM and SIGKILL arrive back-to-back and the
+  handler cannot complete a checkpoint. For preempt jobs the **periodic
+  checkpoint (every K steps) is the ONLY safety net** — this is mandatory, not
+  optional (see §0.2). The trap is dead weight on preemption; it earns its keep
+  only on the walltime path below.
+- On **time limit**, SLURM sends `SIGTERM` at the limit, waits `KillWait` (=30 s
+  here), then `SIGKILL`. To get advance warning *before* the limit, request a
+  signal: `#SBATCH --signal=B:USR1@30` → SLURM delivers `SIGUSR1` to the batch
+  shell ~30 s before the limit. (`B:` = batch shell, not the job steps. `@30`
+  matches the 30 s `KillWait`; raise it only if a checkpoint write needs longer.)
+- So the robust runner traps **SIGUSR1 + SIGTERM + SIGCONT** and routes
+  all three to the same "checkpoint + requeue" handler. This covers preemption
+  *and* walltime on both clusters with one code path.
+
+Required SBATCH directives for any resilient job:
+
+```bash
+#SBATCH --signal=B:USR1@30     # advance warning before the time limit
+#SBATCH --open-mode=append     # preserve logs across requeues (do NOT overwrite)
+#SBATCH --requeue              # mark job requeue-ELIGIBLE (necessary, not sufficient — see §0.1)
+```
+
+## 0.1 Critical: `PreemptMode=REQUEUE` does NOT requeue your job by itself
+
+Verified empirically on this cluster: jobs preempted on `preempt` are **not**
+auto-requeued, despite `PreemptMode=REQUEUE`. Two things defeat the automatic
+path, and you must assume both:
+
+1. **The job must be requeue-eligible.** Without `#SBATCH --requeue` the job is
+   not requeueable, so `PreemptMode=REQUEUE` has nothing to requeue and the job
+   is **cancelled** on preemption. With `GraceTime=0` this is the most likely
+   reason a preempted job vanishes instead of requeuing — it never got a chance
+   to do anything. **`--requeue` is mandatory.**
+2. **A clean exit cancels the requeue** (the walltime path). If your handler ends
+   with `exit 0` *before* SLURM kills the job — which only happens when there IS
+   a grace window, i.e. at the time limit (`KillWait=30`), not on `GraceTime=0`
+   preemption — SLURM records COMPLETED and won't requeue. So at the walltime
+   boundary you must call `scontrol requeue` *explicitly* in the handler.
+3. **Auto-requeue only re-runs the batch script from the top** — it never
+   restores program state. Resume is your code's job regardless (§2), and on
+   preemption the periodic checkpoint (§0.2) is the only thing that survived.
+
+**The fix (what to propose): requeue yourself, explicitly, from the batch
+script — do not depend on `PreemptMode`.** In the trap handler call
+`scontrol requeue "$SLURM_JOB_ID"` *before* the job can exit. Once that call is
+accepted the job is marked for requeue and will restart even if SIGKILL hits a
+moment later — which is why we requeue *first* and checkpoint *second* (a slow
+checkpoint must never cost you the requeue). `#SBATCH --requeue` only makes the
+job *eligible*; the explicit `scontrol requeue` is what actually does it.
+
+**Requeue from bash, checkpoint from Python.** Put the `scontrol requeue` in the
+SLURM batch (bash) layer, not in the training code — bash is the job's top
+process, it can requeue even if Python is wedged, and it keeps the training
+script SLURM-agnostic. Python's only job is: trap `SIGUSR1` → write the latest
+checkpoint → exit. This division is the "whatever is safer" answer: the requeue
+can't be lost to a hung trainer, and the trainer stays portable.
+
+## 0.2 MANDATORY: periodic checkpointing for every preempt job
+
+Because `GraceTime=0`, a preempted job is killed with no warning the handler can
+act on. The **only** state that survives is the last checkpoint the training loop
+wrote on its own schedule. Therefore, for any job on `preempt` (or any job that
+could be requeued), periodic checkpointing is **mandatory**:
+
+- Write a checkpoint every `K` steps such that **at most ~30 min of compute** is
+  ever at risk (tune `K` to the model's step time). This bounds worst-case loss
+  on an unannounced preemption.
+- Overwrite a single latest checkpoint with an **atomic write** (write to
+  `*.tmp`, then `mv` into place) so a kill mid-write can't corrupt the resume
+  point.
+- Delete the checkpoint on clean finish so a completed run leaves no resume bait.
+- The `SIGUSR1` handler's checkpoint is a *best-effort bonus* for the walltime
+  path only; never rely on it for preemption. A preempt job without periodic
+  checkpointing is not resilient — it just hasn't been preempted yet.
+
+Combined with `#SBATCH --requeue`, this is the whole preempt-survival contract:
+**`--requeue` (so SLURM puts it back) + periodic checkpoint (so it resumes near
+where it died)**. The trap adds the walltime case on top.
+
+---
+
+## 0.3 Sizing the grace window AND the walltime (the 2026-06-10 timeout incident)
+
+Two sizing mistakes both end in a dead job; the trap only saves you from the first.
+
+**Grace (`@N` in `--signal=B:USR1@N`) must cover the time to the NEXT checkpoint
+opportunity, not just the checkpoint write.** If the loop can only checkpoint at a
+step/epoch boundary, and a boundary is preceded by a long *uninterruptible* section
+— an eval pass, a validation epoch, a big batch — then `@30` is too short: the
+handler sets the flag, but the flag is not read until that section finishes, and
+SIGKILL lands first. Size `N` ≥ (longest uninterruptible section + checkpoint
+write). Example: the DT-AntMaze runner evals between train steps, so it uses
+`--signal=B:USR1@300`, not `@30`.
+
+**Walltime must be sized to the SLOWEST cell of a sweep — never inherited from a
+faster one.** Incident (2026-06-10): a Meta-LoRA DT sweep set `--time` per *cadence*
+(mild 3:00, medium 3:30, aggressive 5:00) assuming more-frequent resampling = slower.
+But runtime was dominated by the *dataset*: the `umaze-diverse` runs took ~3.0–4.5 h
+regardless of cadence. The 3:00 / 3:30 limits — fine for the ~2× faster non-diverse
+`umaze` grid they were copied from — were ~5 min too short, and **44 of 90 jobs hit
+TIMEOUT** while the 5:00 aggressive cells (same dataset) all finished. The kicker:
+the trainer already had a full SIGUSR1 → checkpoint → resume path, but the SLURM
+template never set `--signal` / `--requeue`, so the trap never fired and the work
+was lost instead of resumed. That is exactly the failure rules 6–7 exist to prevent
+— wire the trap and a too-short walltime becomes a free requeue, not 44 dead jobs.
+
+---
+
+## 1. The trap + checkpoint + requeue handler (CRASH-SAFE walltime default)
+
+**Requeue IF AND ONLY IF: (a) the walltime warning fired AND (b) the run did not
+finish.** Never on a crash, never on a `scancel`, never on a job that completed
+before walltime. User directive (2026-06-10/-12): auto-requeue an overrunning job
+until it finishes, but never resurrect a job that died for another reason — that
+loops a crash forever or fights an operator's `scancel`.
+
+The two discriminators (note: **the exit code is NOT one** — see below):
+
+- **`SIGUSR1` = the walltime advance-warning** from `--signal=B:USR1@N`. ONLY a
+  walltime stop produces it; a crash, a `scancel`, and a `GraceTime=0` preemption do
+  not. So `WALLTIME=1` (set by the USR1 trap) is the authoritative "this is a
+  walltime stop" flag, and it is what excludes crash and cancel.
+- **A completion sentinel** written ONLY on true finish (`metrics.json`, `DONE`, …):
+  present ⇒ finished ⇒ no requeue. This excludes "completed before walltime."
+- **Trap `USR1` + `TERM` + `CONT`.** USR1 sets `WALLTIME=1` and forwards a checkpoint
+  request to the child. TERM/CONT also forward a checkpoint and — by being trapped —
+  keep the shell alive through the hard-kill window so the post-`wait` decision runs
+  instead of the shell dying instantly. A `scancel` is also `TERM`, but it arrives
+  with `WALLTIME=0`, so it is never requeued.
+- **Do NOT gate the requeue on the exit code.** A walltime stop can exit cleanly
+  (the trainer checkpointed and exited 0) *or* be hard-killed non-zero (its checkpoint
+  ran past the grace). Both must requeue. `WALLTIME=1` already excludes crash and
+  cancel, so testing `rc` adds nothing and would wrongly drop the hard-killed case.
+  (For a *generic* job that does NOT self-handle USR1, the equivalent test is "exited
+  non-zero after the walltime warline" — i.e. requeue iff `WALLTIME=1 && rc!=0` and
+  there is no sentinel; a clean `rc==0` then means it completed. Our trainers write a
+  sentinel, so we key on the sentinel, which is unambiguous either way.)
+- `SLURM_RESTART_COUNT` hard-caps the loop (a backstop, not the stop condition).
+
+```bash
+# ---- resilience: crash-safe walltime requeue -----------------------------
+REQUEUE_CAP=8
+WALLTIME=0
+CHILD_PID=""
+SENTINEL="${RUN_DIR}/metrics.json"          # written ONLY on true completion
+
+_ckpt_child() { [ -n "$CHILD_PID" ] && kill -0 "$CHILD_PID" 2>/dev/null && kill -USR1 "$CHILD_PID" 2>/dev/null || true; }
+_on_usr1() { WALLTIME=1; echo "[resilience] SIGUSR1 (walltime) $(date -Is); child -> checkpoint+exit"; _ckpt_child; }
+_on_term() { echo "[resilience] TERM/CONT (walltime=$WALLTIME); forward checkpoint, defer requeue decision"; _ckpt_child; }
+trap _on_usr1 USR1
+trap _on_term TERM CONT                      # keep the shell alive; scancel(WALLTIME=0) won't requeue
+
+python my_script.py --resume-dir "$RUN_DIR" --args ... &
+CHILD_PID=$!
+# wait returns >128 either because a trap interrupted it (child alive — re-wait) OR
+# because the child itself died from a signal (already reaped). Re-wait ONLY while the
+# child is still alive, so a signal-death (SIGKILL/SIGSEGV) cannot spin the loop.
+rc=0; wait "$CHILD_PID" || rc=$?
+while [ "$rc" -gt 128 ] && kill -0 "$CHILD_PID" 2>/dev/null; do rc=0; wait "$CHILD_PID" || rc=$?; done
+
+if [ -f "$SENTINEL" ]; then
+    echo "[resilience] finished before walltime (sentinel, rc=$rc) — no requeue"
+elif [ "$WALLTIME" -eq 1 ]; then            # walltime + unfinished — rc-independent
+    n="${SLURM_RESTART_COUNT:-0}"
+    if [ "$n" -lt "$REQUEUE_CAP" ]; then
+        echo "[resilience] walltime + unfinished (rc=$rc, $n/$REQUEUE_CAP) — requeue"
+        scontrol requeue "$SLURM_JOB_ID"; exit 0
+    fi
+    echo "[resilience] requeue cap reached — FAIL" >&2; exit 1
+else
+    echo "[resilience] rc=$rc walltime=0 no sentinel — crash/cancel/preempt, NO requeue" >&2
+    exit "$([ "$rc" -eq 0 ] && echo 1 || echo "$rc")"
+fi
+# --------------------------------------------------------------------------
+```
+
+The requeue decision is made *after* the child is reaped (not "requeue first") so it
+can see the sentinel and the `WALLTIME` flag together. The slow-checkpoint risk that
+motivates "requeue first" is covered instead by a generous grace (`@300`, §0.3) plus
+trapping `TERM` (which holds the shell open if the hard kill arrives before the child
+finishes). For `GraceTime=0` **preemption** there is no warning at all — see §1b.
+
+## 1b. Preemption addendum (only for `preempt`-partition jobs)
+
+Preemption delivers `SIGTERM`+`SIGKILL` with `GraceTime=0` — no warning, and the
+`SIGTERM` is indistinguishable from a `scancel`. So the §1 walltime handler cannot
+safely requeue a preemption. For preempt jobs the survival contract is instead
+**`#SBATCH --requeue` (eligibility) + periodic checkpoint (§0.2) + resume (§2)**:
+SLURM's `PreemptMode=REQUEUE` puts the eligible job back, and it resumes from the last
+periodic checkpoint. Do not add a `SIGTERM` requeue trap to "improve" this — it buys
+nothing on `GraceTime=0` and reintroduces the scancel-requeue bug.
+
+**Python side** (the child) traps `SIGUSR1`, writes the latest checkpoint, and
+exits non-zero-but-clean (or 0) so the bash handler proceeds:
+
+```python
+import signal, sys
+_should_stop = False
+def _on_usr1(signum, frame):
+    global _should_stop
+    _should_stop = True
+signal.signal(signal.SIGUSR1, _on_usr1)
+# ... in the training loop, after each step/epoch:
+if _should_stop:
+    save_checkpoint(run_dir)   # latest only; overwrite previous
+    sys.exit(0)
+```
+
+If your training framework can't be interrupted mid-step, the simplest robust
+form is: checkpoint every K steps unconditionally, and on `SIGUSR1` just set the
+flag and let the next step boundary save+exit. The grace window only needs to
+cover one checkpoint write.
+
+---
+
+## 2. Resume on restart (robust, sentinel-independent)
+
+When the requeued job starts, it must find the latest checkpoint and resume.
+**Do not rely on a single `last_run_dir.txt` sentinel** — that has silently
+gone stale after multiple requeues on this cluster, causing a fresh-from-zero
+restart that abandons a near-finished run. Probe the filesystem directly for the
+highest-step checkpoint:
+
+```bash
+# Prefer an explicit override; else find the newest checkpoint by step number.
+RUN_DIR="${RESUME_RUN_DIR:-}"
+if [ -z "$RUN_DIR" ]; then
+    latest_ckpt=$(ls -1 "$OUT_ROOT"/run_*/checkpoints*/checkpoint_*.pt 2>/dev/null \
+        | sed -E 's/.*checkpoint_[a-z]*([0-9]+)\.pt/\1 &/' \
+        | sort -n | tail -1 | cut -d' ' -f2-)
+    [ -n "$latest_ckpt" ] && RUN_DIR=$(dirname "$(dirname "$latest_ckpt")")
+fi
+if [ -n "$RUN_DIR" ] && [ -d "$RUN_DIR" ]; then
+    echo "[resilience] resuming from $RUN_DIR"
+else
+    RUN_DIR="$OUT_ROOT/run_$(date +%s)"; mkdir -p "$RUN_DIR"
+    echo "[resilience] no checkpoint found; starting fresh in $RUN_DIR"
+fi
+```
+
+- Always honor a manual `RESUME_RUN_DIR=<dir>` env override — it's the escape
+  hatch when automatic detection picks wrong.
+- Keep only the latest checkpoint (overwrite), and **delete it on clean finish**
+  so a finished run doesn't leave resume bait or waste disk.
+- `--open-mode=append` (set in the submitter) keeps every requeue's log in one
+  file so the failure trail survives.
+
+---
+
+## 3. VRAM guard — auto-requeue off a too-small GPU
+
+Even with a VRAM `--constraint`, add this backstop for any A100 / mixed-VRAM
+request. At job start the runner checks the card it actually got; if it's below
+the requirement it excludes that node and requeues. Over repeated requeues the
+job walks itself onto a correct card.
+
+```bash
+MIN_GPU_MEM_MB="${MIN_GPU_MEM_MB:-80000}"   # set to the model's real need
+gpu_mem=$(nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits 2>/dev/null | head -1)
+if [ -n "$gpu_mem" ] && [ "$gpu_mem" -lt "$MIN_GPU_MEM_MB" ]; then
+    echo "[vram-guard] landed on ${gpu_mem} MiB < required ${MIN_GPU_MEM_MB} on ${SLURMD_NODENAME}; excluding + requeuing"
+    _exclude_node_and_requeue "$SLURMD_NODENAME"   # see §4
+    exit 0
+fi
+echo "[vram-guard] GPU OK: ${gpu_mem} MiB on ${SLURMD_NODENAME}"
+```
+
+Set `MIN_GPU_MEM_MB` to match the constraint — don't claim `a100-80G` but set
+`MIN_GPU_MEM_MB=40000`, or the guard goes slack and you get confusing late OOMs.
+
+---
+
+## 4. Adaptive exclude — start empty, learn at runtime
+
+**Submitters set NO `--exclude`.** Each submission re-tests every node (broken
+nodes do get fixed). The runner discovers a bad node — wrong VRAM (§3), a
+`libtorch_global_deps.so: cannot open shared object file` NFS error, a CUDA init
+failure — and requeues itself with that node appended to a **job-scoped**
+exclude list via `ExcNodeList`:
+
+```bash
+_exclude_node_and_requeue() {
+    local bad="$1"
+    # Accumulate across this job's requeues in a small per-job file.
+    local exfile="${EXCLUDE_FILE:-$OUT_ROOT/.exclude_${SLURM_JOB_ID}}"
+    echo "$bad" >> "$exfile"
+    local list; list=$(sort -u "$exfile" | paste -sd, -)
+    echo "[exclude] requeuing ${SLURM_JOB_ID} with ExcNodeList=${list}"
+    scontrol requeue "$SLURM_JOB_ID"
+    scontrol update JobId="$SLURM_JOB_ID" ExcNodeList="$list"
+}
+```
+
+This is strictly better than a hardcoded list: it's current (re-tests on every
+fresh submission), self-healing (walks off a broken node in seconds), and leaves
+no stale exclusions to silently shrink the pool months later.
+
+Guard the runner's startup with a quick sanity import so a node-level breakage is
+caught *before* training starts:
+
+```bash
+python -c "import torch; assert torch.cuda.is_available()" \
+    || _exclude_node_and_requeue "$SLURMD_NODENAME"
+```
+
+---
+
+## 5. Walltime self-requeue (universal — until the job is DONE)
+
+**This is mandatory on every job, regardless of expected runtime** (rule 7).
+The same trap handles the cap: `--signal=B:USR1@30` fires ~30 s before the
+`--time` limit, the handler checkpoints + `scontrol requeue`s, and the requeued
+job resumes (§2). So a run that overruns one allocation just needs `--time` set,
+the trap wired, and a working resume — it will tick across as many requeues as
+needed.
+
+**"Until completed" needs a stop condition.** Auto-requeue must end when the
+work is finished, not loop forever after a clean finish or a non-walltime crash.
+Gate it on a `DONE` sentinel the training code writes on successful completion:
+resume while `DONE` is absent, and skip the trap/requeue once it exists.
+
+```bash
+# at the top of the runner, before launching training:
+if [ -f "$RUN_DIR/DONE" ]; then
+    echo "[resilience] $RUN_DIR already DONE; nothing to requeue"; exit 0
+fi
+# ... trap _on_signal USR1 TERM CONT ; run training ...
+# the trainer writes "$RUN_DIR/DONE" (atomically) only on successful completion.
+# the _on_signal handler should no-op if DONE already exists:
+#   _on_signal() { [ -f "$RUN_DIR/DONE" ] && exit 0; ... ; scontrol requeue ...; }
+```
+
+Verify the checkpoint cadence is short enough that losing the work since the
+last checkpoint is acceptable (e.g. checkpoint every ≤30 min of compute).
+
+For the **new cluster's 2-day cap specifically**: set `--time=2-00:00:00`, wire
+the trap, and the job auto-requeues + resumes at the boundary. (There is a
+`normal-7days` QOS on the new cluster but it has **no GPU** — it can't substitute
+for resume on GPU work.)
+
+---
+
+## 6. Splitting big tasks — dependency chains
+
+When a task is naturally staged, or you'd rather split a long run into bounded
+segments than rely on in-place requeue, chain jobs with `--dependency`:
+
+```bash
+jid1=$(sbatch --parsable stage1.sbatch)                                  # e.g. preprocess
+jid2=$(sbatch --parsable --dependency=afterok:$jid1   stage2.sbatch)     # train, runs iff stage1 ok
+jid3=$(sbatch --parsable --dependency=afterok:$jid2   stage3.sbatch)     # eval/package
+```
+
+Dependency types:
+- `afterok:JID` — start only if JID succeeded (exit 0).
+- `afterany:JID` — start regardless of how JID ended. Use for a **resume
+  segment**: segment N+1 should run even if segment N hit the walltime (non-zero
+  exit), picking up from the checkpoint.
+- `afternotok:JID` — start only if JID failed (cleanup / fallback paths).
+- `singleton` — serialize all jobs sharing the same `--job-name` + user; a clean
+  way to make a self-chaining resume job never run two copies at once.
+
+Self-chaining segment pattern (each segment trains for ~1 cap, then submits its
+own successor before exiting, conditioned on "not yet converged"):
+
+```bash
+# near the end of the runner, if training is incomplete:
+if [ ! -f "$RUN_DIR/DONE" ]; then
+    sbatch --parsable --dependency=afterany:${SLURM_JOB_ID} "$0"   # queue the next segment
+fi
+```
+
+Prefer in-place requeue (§1–§2) for a single long run; prefer dependency chains
+when stages have distinct resource shapes (CPU preprocess → GPU train → CPU
+package) or when you want each segment's logs and accounting separated.
+
+---
+
+## 7. Putting it together — minimal resilient GPU runner skeleton
+
+```bash
+#!/usr/bin/env bash
+#SBATCH --job-name=myrun
+#SBATCH --output=./res_%j_%N.txt
+#SBATCH --error=./res_%j_%N.err
+#SBATCH --time=2-00:00:00
+#SBATCH --partition=preempt
+#SBATCH --gres=gpu:1
+#SBATCH --constraint=a100-80G
+#SBATCH --cpus-per-task=8
+#SBATCH --mem=64G
+#SBATCH --signal=B:USR1@30
+#SBATCH --open-mode=append
+#SBATCH --requeue
+
+set -uo pipefail
+OUT_ROOT=/cluster/tufts/levinlab/hhazan01/PROJECT/out
+MIN_GPU_MEM_MB=80000
+
+source ~/.bashrc && conda activate LoRa          # NEW cluster
+# (OLD cluster: module load singularity; wrap the python line in singularity exec)
+
+# --- instrumentation: which node/GPU did we actually get? (see ../SKILL.md) ---
+echo "===== START $(date -Is)  node=${SLURMD_NODENAME} job=${SLURM_JOB_ID} ====="
+nvidia-smi --query-gpu=name,memory.total --format=csv,noheader || true
+
+# --- §4 startup sanity + §3 VRAM guard ---
+# ( _exclude_node_and_requeue + _on_signal defined as above )
+python -c "import torch; assert torch.cuda.is_available()" || _exclude_node_and_requeue "$SLURMD_NODENAME"
+gpu_mem=$(nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits | head -1)
+[ "${gpu_mem:-0}" -lt "$MIN_GPU_MEM_MB" ] && { _exclude_node_and_requeue "$SLURMD_NODENAME"; exit 0; }
+
+# --- §2 resolve resume dir ---
+# ( RUN_DIR resolution block as above )
+
+# --- §1 trap + launch ---
+trap _on_signal USR1 TERM CONT
+python my_script.py --resume-dir "$RUN_DIR" --args ... &
+CHILD_PID=$!
+wait "$CHILD_PID"
+echo "===== END $(date -Is)  exit=$? ====="
+```
+
+This single runner is preempt-safe, walltime-safe, lands on a correct-VRAM GPU
+or requeues, learns bad nodes with no hardcoded exclude, and resumes from its own
+checkpoint — the full set of guarantees the user asked for.
diff --git a/.claude/skills/report/SKILL.md b/.claude/skills/report/SKILL.md
new file mode 100644
index 000000000..f1afd0c53
--- /dev/null
+++ b/.claude/skills/report/SKILL.md
@@ -0,0 +1,53 @@
+---
+name: report
+description: Plain-English status/results report for any project. Use whenever the user
+  asks for "a report", "what ran", "status", "what happened", "summary", "explain the
+  results", or wants work explained simply for a non-specialist. Reads the project's own
+  durable records (logs, status files, trackers, git history).
+---
+
+# Plain-English report
+
+The reader is smart and busy, wants the truth fast, and hates jargon. Write so a
+non-specialist on this project can follow it.
+
+## Rules
+- Plain English. Short sentences, common words. If a technical term is unavoidable,
+  define it in the same breath, once (e.g. "gated = run through a strict pass/fail test").
+- Facts only. Every claim ties to a source: a run/commit id, a number, a file, a ticket.
+  Never invent. If something is unknown, say "unknown".
+- No spin. Negative, null, and failed results get equal weight. If an earlier claim was
+  reversed, say so plainly under Corrections.
+- Brief. Group many items by theme; headline per theme, not a play-by-play.
+
+## Figures & diagrams (a picture often beats a paragraph)
+When a figure makes the report clearer, include one — and add a one-line caption saying
+the single thing it shows:
+- Concept / architecture / flow / decision -> an ASCII diagram inline.
+- Data / results (trends, comparisons, distributions) -> generate a REAL plot from the
+  data with a small script (never hand-draw numbers); save it under the project's figures
+  area and reference it. Follow any "figures regenerated from data" rule the project has.
+- Prefer a figure over a long paragraph when it compresses the idea. Skip decorative ones;
+  every figure must earn its place.
+
+## Find the sources first (use whatever the project actually keeps)
+experiment logs, LOG/STATE/STATUS/CHANGELOG/NOTES, lab notebooks; trackers / results
+tables (csv, json, dashboards); recent git commits, open PRs/issues, CI; README/docs for
+baseline and goal. For large reads use a subagent; bring back only conclusions. If there
+is no durable record, say so and report from what's observable.
+
+## Format
+One opening line: the big-picture arc.
+Then per theme (or per item if asked):
+- Task — what was tried, one sentence.
+- Baseline — the number/behaviour being compared against.
+- Result — real numbers (value ± spread, sample size, conditions); a figure if it helps.
+- What it means — the takeaway, plain.
+- How we know — why it's trustworthy (method, controls, sample size, fair test).
+Close with:
+- Still running / open — each live item: where it is and when it lands.
+- Corrections — anything that reversed an earlier belief (integrity first).
+
+## Scope
+Honor any window ("last 3 days") or sub-area named. Default: since the last report, or
+the currently-active work.
\ No newline at end of file
diff --git a/.claude/skills/slurm-harvest/SKILL.md b/.claude/skills/slurm-harvest/SKILL.md
new file mode 100644
index 000000000..5f54a743b
--- /dev/null
+++ b/.claude/skills/slurm-harvest/SKILL.md
@@ -0,0 +1,38 @@
+---
+name: harvest-slurm
+description: Extract durable compute provenance (node, GPU/CPU specs, wall time, job ids, fitness/result lines) from SLURM stdout/stderr logs into research/results/compute_provenance.md, so the raw per-task logs can be deleted without losing reproducibility metadata. Use when a run's SLURM logs pile up and you want the proof-of-run kept in the research record, not as thousands of scheduler files.
+user-invocable: true
+disable-model-invocation: false
+argument-hint: [glob of slurm logs] [run-id]
+---
+
+# Harvest SLURM logs → compute provenance
+
+SLURM job logs (`*.out`/`*.err`, `slurm-*.out`, array logs) are scheduler noise, not
+research records — but they carry real proof-of-run: node, GPU model + memory, gpu/cpu
+counts, start/wall time, job ids, and any fitness/result lines the job printed. This skill
+harvests those facts into `research/results/compute_provenance.md` (keyed by LOG run-id),
+after which the raw logs are safe to delete.
+
+## Procedure
+
+1. **Identify the log group** and the LOG run-id it belongs to (map job-name → run via
+   `research/LOG.md`). Group by job family if a run produced several (train + eval array, ES
+   coord + workers, etc.).
+2. **Harvest** (prints a Markdown block; add `--json out.json` for machine-readable):
+   ```
+   python .claude/skills/harvest-slurm/harvest_slurm.py --md \
+     --label "<run name>" --run-id <YYYY-MM-DD-rNN> --glob '<path/to/*.out>'
+   ```
+   It extracts: node(s), GPU model+MiB, gpu/cpu per job, start times, per-cell wall seconds
+   (summed), ES per-gen fitness trajectory, aggregate fitness units (n/mean/range), job ids,
+   and a count of files with error/traceback markers. Absent fields are simply omitted.
+3. **Append the block** to `research/results/compute_provenance.md` under the right run-id.
+   Note in it that the raw logs were harvested then deleted.
+4. **Delete the raw logs** once the provenance is captured. Do not add them to `.gitignore`
+   — the workflow no longer dumps logs into the repo; if any reappear, harvest + delete again.
+5. **Sanity-check** the harvested numbers against the run's own result JSON / LOG entry
+   (e.g. ES fitness trajectory should match the recorded verdict). Flag discrepancies.
+
+The extractor is pure-stdlib regex over heterogeneous formats; if a new log family prints
+specs differently, extend the patterns in `harvest_slurm.py`.
diff --git a/.claude/skills/slurm-harvest/harvest_slurm.py b/.claude/skills/slurm-harvest/harvest_slurm.py
new file mode 100644
index 000000000..564162f17
--- /dev/null
+++ b/.claude/skills/slurm-harvest/harvest_slurm.py
@@ -0,0 +1,189 @@
+#!/usr/bin/env python3
+"""Harvest compute provenance from SLURM stdout/stderr logs, then they can be deleted.
+
+Scans a set of SLURM log files and extracts the durable "proof of run" facts —
+node(s), GPU model + memory, gpu/cpu counts, start time, wall time, job ids, and a
+sample of result/fitness lines — into a compact JSON + Markdown summary. The summary
+is meant to live in the research record (e.g. research/results/compute_provenance.md)
+so the raw per-task logs can be removed without losing reproducibility metadata.
+
+Pure stdlib. Robust to heterogeneous formats (training logs, ES coord/eval, eval
+arrays, probe units) via regex signal extraction — absent fields are simply omitted.
+
+Usage:
+    python harvest_slurm.py --label "ES run (E009)" --run-id 2026-06-03-r01 \
+        --glob 'scripts/stage_b/slurm_logs/*.out' [--json out.json] [--md]
+"""
+from __future__ import annotations
+
+import argparse
+import glob as globmod
+import json
+import re
+import sys
+from collections import Counter
+
+# --- signal patterns (lenient; match whatever a given log family happens to print)
+RE_GPU = re.compile(r"(NVIDIA[\w .\-]*?,\s*\d+\s*MiB|NVIDIA[\w .\-]+-\d+GB)")
+RE_NODE = re.compile(
+    r"\bnode:\s*([\w\-]+)|on\s+([\w\-]+)\s*=*$|coordinator:.*node:\s*([\w\-]+)"
+)
+RE_NODE2 = re.compile(r"\bnode:\s*([\w\-]+)")
+RE_GPUS = re.compile(r"\bgpus:\s*(\d+)")
+RE_CPUS = re.compile(r"\bcpus:\s*(\d+)")
+RE_STARTED = re.compile(r"\bstarted:\s*(.+)$")
+RE_JOB = re.compile(r"\bjob:\s*(\d+)")
+RE_CELL_SECS = re.compile(r"\((\d+)s\)")  # eval array: "(887s)"
+RE_FIT_COORD = re.compile(r"coord gen (\d+):\s*fit_mean=([-\d.]+)\s*fit_max=([-\d.]+)")
+RE_FITNESS = re.compile(r"FITNESS\s+mean_per_cog=([-\d.]+)")
+RE_ERROR = re.compile(
+    r"\b(Traceback|Error|CUDA out of memory|Killed|oom-kill|FAILED)\b", re.I
+)
+
+
+def harvest(paths: list[str]) -> dict:
+    nodes, gpus_models, jobs = Counter(), Counter(), set()
+    gpu_count = cpu_count = None
+    starts: list[str] = []
+    cell_secs: list[int] = []
+    coord_series: list[tuple[int, float, float]] = []
+    fitness_vals: list[float] = []
+    errors = 0
+    n = 0
+    for p in paths:
+        n += 1
+        try:
+            text = open(p, errors="replace").read()
+        except OSError:
+            continue
+        for m in RE_GPU.finditer(text):
+            gpus_models[m.group(1).strip()] += 1
+        for m in RE_NODE2.finditer(text):
+            nodes[m.group(1)] += 1
+        if g := RE_GPUS.search(text):
+            gpu_count = g.group(1)
+        if c := RE_CPUS.search(text):
+            cpu_count = c.group(1)
+        for m in RE_STARTED.finditer(text):
+            starts.append(m.group(1).strip())
+        for m in RE_JOB.finditer(text):
+            jobs.add(m.group(1))
+        cell_secs += [int(s) for s in RE_CELL_SECS.findall(text)]
+        for m in RE_FIT_COORD.finditer(text):
+            coord_series.append((int(m.group(1)), float(m.group(2)), float(m.group(3))))
+        fitness_vals += [float(v) for v in RE_FITNESS.findall(text)]
+        if RE_ERROR.search(text):
+            errors += 1
+
+    out: dict = {"n_files": n}
+    if nodes:
+        out["nodes"] = sorted(nodes)
+    if gpus_models:
+        out["gpu_models"] = sorted(gpus_models)
+    if gpu_count:
+        out["gpus_per_job"] = gpu_count
+    if cpu_count:
+        out["cpus_per_job"] = cpu_count
+    if starts:
+        out["started"] = sorted(set(starts))[:4]
+    if jobs:
+        out["job_ids_sample"] = sorted(jobs)[:8]
+        out["n_job_ids"] = len(jobs)
+    if cell_secs:
+        out["cell_wall_seconds"] = {
+            "n": len(cell_secs),
+            "sum_s": sum(cell_secs),
+            "sum_h": round(sum(cell_secs) / 3600, 2),
+            "max_s": max(cell_secs),
+        }
+    if coord_series:
+        coord_series.sort()
+        out["es_fitness_by_gen"] = [
+            {"gen": g, "fit_mean": fm, "fit_max": fx} for g, fm, fx in coord_series
+        ]
+    if fitness_vals:
+        out["fitness_units"] = {
+            "n": len(fitness_vals),
+            "mean": round(sum(fitness_vals) / len(fitness_vals), 4),
+            "min": min(fitness_vals),
+            "max": max(fitness_vals),
+        }
+    if errors:
+        out["files_with_error_markers"] = errors
+    return out
+
+
+def to_md(label: str, run_id: str, src: str, h: dict) -> str:
+    lines = [
+        f"### {label}" + (f"  (run {run_id})" if run_id else ""),
+        f"- Source logs: `{src}` — {h.get('n_files', 0)} files (harvested, then deleted).",
+    ]
+    if "gpu_models" in h:
+        lines.append(
+            f"- GPU: {', '.join(h['gpu_models'])}"
+            + (
+                f"; {h['gpus_per_job']} gpu / {h.get('cpus_per_job','?')} cpu per job"
+                if "gpus_per_job" in h
+                else ""
+            )
+        )
+    if "nodes" in h:
+        ns = h["nodes"]
+        lines.append(f"- Node(s): {', '.join(ns[:8])}" + (" …" if len(ns) > 8 else ""))
+    if "started" in h:
+        lines.append(f"- Started: {'; '.join(h['started'])}")
+    if "cell_wall_seconds" in h:
+        cw = h["cell_wall_seconds"]
+        lines.append(
+            f"- Eval wall: {cw['n']} cells, Σ {cw['sum_h']} h (max cell {cw['max_s']} s)."
+        )
+    if "fitness_units" in h:
+        fu = h["fitness_units"]
+        lines.append(
+            f"- Fitness units: n={fu['n']}, mean={fu['mean']}, range [{fu['min']}, {fu['max']}]."
+        )
+    if "es_fitness_by_gen" in h:
+        s = h["es_fitness_by_gen"]
+        traj = ", ".join(f"g{d['gen']}:{d['fit_mean']:.3f}" for d in s)
+        lines.append(f"- ES fitness/gen ({len(s)} gens): {traj}")
+    if "n_job_ids" in h:
+        lines.append(
+            f"- SLURM job ids: {h['n_job_ids']} (e.g. {', '.join(h['job_ids_sample'])})."
+        )
+    if "files_with_error_markers" in h:
+        lines.append(
+            f"- ⚠ {h['files_with_error_markers']} file(s) contained error/traceback markers."
+        )
+    return "\n".join(lines)
+
+
+def main(argv=None) -> int:
+    ap = argparse.ArgumentParser(description=__doc__)
+    ap.add_argument("--glob", required=True, help="glob for the log files")
+    ap.add_argument("--label", required=True)
+    ap.add_argument("--run-id", default="")
+    ap.add_argument("--json", type=str, default="")
+    ap.add_argument("--md", action="store_true", help="print a Markdown block")
+    args = ap.parse_args(argv)
+
+    paths = sorted(globmod.glob(args.glob))
+    if not paths:
+        print(f"no files matched: {args.glob}", file=sys.stderr)
+        return 1
+    h = harvest(paths)
+    if args.json:
+        json.dump(
+            {"label": args.label, "run_id": args.run_id, "glob": args.glob, **h},
+            open(args.json, "w"),
+            indent=2,
+        )
+    print(
+        to_md(args.label, args.run_id, args.glob, h)
+        if args.md
+        else json.dumps(h, indent=2)
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
diff --git a/bindsnet/datasets/torchvision_wrapper.py b/bindsnet/datasets/torchvision_wrapper.py
index b1e36519e..23960a04f 100644
--- a/bindsnet/datasets/torchvision_wrapper.py
+++ b/bindsnet/datasets/torchvision_wrapper.py
@@ -16,13 +16,18 @@ def create_torchvision_dataset_wrapper(ds_type):
         ds_type = getattr(torchDB, ds_type)
 
     class TorchvisionDatasetWrapper(ds_type):
-        __doc__ = """BindsNET torchvision dataset wrapper for:
+        __doc__ = (
+            """BindsNET torchvision dataset wrapper for:
 
         The core difference is the output of __getitem__ is no longer
         (image, label) rather a dictionary containing the image, label,
         and their encoded versions if encoders were provided.
 
-            \n\n""" + str(ds_type) if ds_type.__doc__ is None else ds_type.__doc__
+            \n\n"""
+            + str(ds_type)
+            if ds_type.__doc__ is None
+            else ds_type.__doc__
+        )
 
         def __init__(
             self,
diff --git a/examples/benchmark/sparse_compute and foldable pipeline/_bench_common.py b/examples/benchmark/sparse_compute and foldable pipeline/_bench_common.py
index 564de6591..90811f3ee 100644
--- a/examples/benchmark/sparse_compute and foldable pipeline/_bench_common.py	
+++ b/examples/benchmark/sparse_compute and foldable pipeline/_bench_common.py	
@@ -54,7 +54,11 @@ def _expansion_compute(self, s):
         if isinstance(f, Probability):
             v = torch.bernoulli(f.value)
         elif isinstance(f, Degradation):
-            v = f.degrade_function(f.value) if f.degrade_function is not None else f.value
+            v = (
+                f.degrade_function(f.value)
+                if f.degrade_function is not None
+                else f.value
+            )
         else:
             v = f.value
         if op == "mul":
diff --git a/examples/benchmark/sparse_compute and foldable pipeline/foldable_pipelines.py b/examples/benchmark/sparse_compute and foldable pipeline/foldable_pipelines.py
index ea43adfbe..8295abaa2 100644
--- a/examples/benchmark/sparse_compute and foldable pipeline/foldable_pipelines.py	
+++ b/examples/benchmark/sparse_compute and foldable pipeline/foldable_pipelines.py	
@@ -4,4 +4,6 @@
 # Measures the speedup of the folded MultiCompartmentConnection compute (``out = s @ A + B.sum(0)``) over
 # the pre-fold ``[batch, src, tgt]`` expansion path, on the ExampleNetwork
 if __name__ == "__main__":
-    run_speedup(baseline_mode="expansion", test_mode="fold", technique="Foldable pipelines")
+    run_speedup(
+        baseline_mode="expansion", test_mode="fold", technique="Foldable pipelines"
+    )
diff --git a/examples/benchmark/sparse_compute and foldable pipeline/sparse_compute.py b/examples/benchmark/sparse_compute and foldable pipeline/sparse_compute.py
index 3a4a6f733..5781e8918 100644
--- a/examples/benchmark/sparse_compute and foldable pipeline/sparse_compute.py	
+++ b/examples/benchmark/sparse_compute and foldable pipeline/sparse_compute.py	
@@ -5,4 +5,6 @@
 # only the weight rows of source neurons that spiked) over the dense folded compute,
 # on the ExampleNetwork (CPU and GPU).
 if __name__ == "__main__":
-    run_speedup(baseline_mode="fold", test_mode="fold_sparse", technique="Sparse compute")
+    run_speedup(
+        baseline_mode="fold", test_mode="fold_sparse", technique="Sparse compute"
+    )
diff --git a/examples/stress_test/example_network.py b/examples/stress_test/example_network.py
index 7562a2736..3d8e94f97 100644
--- a/examples/stress_test/example_network.py
+++ b/examples/stress_test/example_network.py
@@ -143,9 +143,7 @@ def build(self):
     def make_input(self, runtime):
         # Poisson-ish random spike train into the input layer.
         return {
-            "I": torch.rand(
-                runtime, self.batch_size, self.in_size, device=self.device
-            )
+            "I": torch.rand(runtime, self.batch_size, self.in_size, device=self.device)
             > 0.90
         }
 
diff --git a/test/network/test_connections.py b/test/network/test_connections.py
index 416855359..f692f0885 100644
--- a/test/network/test_connections.py
+++ b/test/network/test_connections.py
@@ -195,7 +195,7 @@ def _make_mcc(self, pipeline, src_n, tgt_n, batch=1, sparse_compute=False):
         return conn
 
     def _reference_expansion(self, pipeline, s, tgt_n):
-        """Pre-collapse pipeline features """
+        """Pre-collapse pipeline features"""
         b, src = s.shape
         x = s.view(b, src, 1).expand(b, src, tgt_n).clone().float()
         for f in pipeline:
@@ -222,7 +222,11 @@ def _learning_conn(self, rule, w0, nu, rng=(-1.0, 1.0)):
         """MCC with a single learnable Weight; returns (connection, feature)."""
         src_n, tgt_n = w0.shape
         conn = self._make_mcc(
-            [tf.Weight(name="w", value=w0.clone(), learning_rule=rule, nu=nu, range=rng)],
+            [
+                tf.Weight(
+                    name="w", value=w0.clone(), learning_rule=rule, nu=nu, range=rng
+                )
+            ],
             src_n,
             tgt_n,
             batch=1,
@@ -237,7 +241,9 @@ def _mstdp_seq(self):
             (torch.tensor([0.0, 0.0, 0.0]), torch.tensor([0.0, 0.0]), 1.0),
         ]
 
-    def _mstdp_reference(self, w0, seq, nu0, dt, tc_plus=20.0, tc_minus=20.0, rng=(-1.0, 1.0)):
+    def _mstdp_reference(
+        self, w0, seq, nu0, dt, tc_plus=20.0, tc_minus=20.0, rng=(-1.0, 1.0)
+    ):
         w = w0.clone().float()
         src_n, tgt_n = w.shape
         p_plus, p_minus = torch.zeros(src_n), torch.zeros(tgt_n)
@@ -283,7 +289,10 @@ def test_mask_feature_output(self):
         w = torch.tensor([[0.1, 0.2], [0.3, 0.4], [0.5, 0.6]])
         mask = torch.tensor([[True, False], [True, True], [False, True]])
         conn = self._make_mcc(
-            [tf.Weight(name="w", value=w.clone()), tf.Mask(name="m", value=mask.clone())],
+            [
+                tf.Weight(name="w", value=w.clone()),
+                tf.Mask(name="m", value=mask.clone()),
+            ],
             3,
             2,
         )
@@ -295,7 +304,10 @@ def test_bias_feature_output(self):
         w = torch.tensor([[0.1, 0.2], [0.3, 0.4], [0.5, 0.6]])
         bias = torch.tensor([[0.5, -0.5], [0.1, 0.2], [0.0, 1.0]])
         conn = self._make_mcc(
-            [tf.Weight(name="w", value=w.clone()), tf.Bias(name="b", value=bias.clone())],
+            [
+                tf.Weight(name="w", value=w.clone()),
+                tf.Bias(name="b", value=bias.clone()),
+            ],
             3,
             2,
             batch=2,
@@ -496,7 +508,9 @@ def test_postpre_predictable(self):
     def test_learning_respects_range_clamp(self):
         # Post-only potentiation of +5 per synapse must clamp to the range max (1.0).
         w0 = torch.full((2, 2), 0.9)
-        conn, feat = self._learning_conn(mcc.PostPre, w0, nu=(0.0, 5.0), rng=(-1.0, 1.0))
+        conn, feat = self._learning_conn(
+            mcc.PostPre, w0, nu=(0.0, 5.0), rng=(-1.0, 1.0)
+        )
         conn.source.s = torch.zeros(1, 2)
         conn.target.s = torch.ones(1, 2)
         conn.source.x = torch.ones(1, 2)

From b0b055b5d9e014595c7b120940bef9d2c72beb82 Mon Sep 17 00:00:00 2001
From: Hananel Hazan <Hananel@Hazan.org.il>
Date: Mon, 3 Aug 2026 20:52:52 -0400
Subject: [PATCH 10/14] Correctness fixes for the folded pipeline
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- test_connections.py: drop __init__ constructors so pytest actually
  collects the classes (0 tests ran before); fix test_transfer to move
  the connection to the device before asserting placement; rename the
  argument-taking test_weights to check_weights (manual matrix check,
  driven from __main__); make the __main__ MCC list self-discovering.
- topology.py: densify sparse feature values ([1, src, tgt] from
  prime_feature) into [src, tgt] fold factors — Probability(sparse=True)
  crashed with "expand is unsupported for Sparse tensors"; restore
  empty-pipeline fan-in behavior; skip None factors from sub-features;
  gate sparse_compute per device (CUDA needs >= 4e6 synapses to amortize
  the nonzero() sync; on CPU the gather helps even at small sizes).
- topology_features.py: sub-features return None (identity in the fold)
  instead of int 1.
- MCC_learning.py: key the cached MSTDP decay factors on (dt, device) so
  changing network.dt or moving devices recomputes them instead of
  silently reusing stale values.
- _bench_common.py: repo root is three levels up, not two — the
  benchmarks were unimportable without an installed bindsnet.
- New regression tests: sparse Probability (dense + gather paths), empty
  pipeline, sub-feature identity, MSTDP dt change, CUDA gate equality.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 bindsnet/learning/MCC_learning.py             |  10 +-
 bindsnet/network/topology.py                  |  22 ++-
 bindsnet/network/topology_features.py         |   4 +-
 .../_bench_common.py                          |   7 +-
 test/network/test_connections.py              | 133 ++++++++++++++----
 5 files changed, 134 insertions(+), 42 deletions(-)

diff --git a/bindsnet/learning/MCC_learning.py b/bindsnet/learning/MCC_learning.py
index 9c2a7b781..fd26dca15 100644
--- a/bindsnet/learning/MCC_learning.py
+++ b/bindsnet/learning/MCC_learning.py
@@ -509,12 +509,14 @@ def _connection_update(self, **kwargs) -> None:
         # Reward from current time step
         reward = kwargs["reward"]
 
-        # Build learning-rate and decay tensors
-        if not hasattr(self, "_a_plus_default"):
-            dev = self.feature_value.device
+        # Build learning-rate and decay tensors, cached on (dt, device) so a
+        # change to either (network.dt edits, .to(device) moves) recomputes them.
+        dev = self.feature_value.device
+        dt = float(self.connection.dt)
+        if getattr(self, "_decay_key", None) != (dt, dev):
+            self._decay_key = (dt, dev)
             self._a_plus_default = torch.tensor(1.0, device=dev)
             self._a_minus_default = torch.tensor(-1.0, device=dev)
-            dt = float(self.connection.dt)
             self._decay_plus = torch.exp(-dt / self.tc_plus.to(dev))
             self._decay_minus = torch.exp(-dt / self.tc_minus.to(dev))
         a_plus = kwargs.get("a_plus", None)
diff --git a/bindsnet/network/topology.py b/bindsnet/network/topology.py
index eaa4485f8..f64227f26 100644
--- a/bindsnet/network/topology.py
+++ b/bindsnet/network/topology.py
@@ -428,7 +428,10 @@ def __init__(
             False by default, updates called after each time step
         :param traces: Set to :code:`True` to record history of connection activity (for monitors)
         :param sparse_compute: Set to :code:`True` to read only the rows of the effective
-            weight for source neurons that spiked (a win only when few are active).
+            weight for source neurons that spiked. A win when few sources are active;
+            on CUDA it is applied only for large connections
+            (``source.n * target.n >= 4e6``), where the required device sync pays
+            for itself. Ignored otherwise.
         """
 
         super().__init__(source, target, device, pipeline, **kwargs)
@@ -466,6 +469,13 @@ def compute(self, s: torch.Tensor) -> torch.Tensor:
         b_eff = None
         for f in self.pipeline:
             factor = f.compute(s)
+            if factor is None:
+                # Side-effect-only pipeline entries (sub-features) fold as identity.
+                continue
+            if isinstance(factor, torch.Tensor) and factor.is_sparse:
+                # Sparse feature values carry a leading batch dim ([1, src, tgt])
+                # from prime_feature; densify to the fold's [src, tgt] shape.
+                factor = factor.to_dense().view(self.source.n, self.target.n)
             op = getattr(f, "op", "mul")
             if op == "mul":
                 a_eff = factor if a_eff is None else a_eff * factor
@@ -478,11 +488,17 @@ def compute(self, s: torch.Tensor) -> torch.Tensor:
         if a_eff is None:
             # Degenerate pipeline with no multiplicative feature: every source
             # neuron contributes with unit weight.
-            a_eff = torch.ones(s.size(1), b_eff.size(-1), device=s.device)
+            a_eff = torch.ones(self.source.n, self.target.n, device=s.device)
         if not torch.is_floating_point(a_eff):
             a_eff = a_eff.float()
 
-        if self.sparse_compute:
+        # The gather pays for its ``nonzero()`` only where that sync is expensive
+        # relative to the matmul: on CUDA it needs a large weight matrix to win;
+        # on CPU there is no device sync, so it helps even at small sizes.
+        use_gather = self.sparse_compute and (
+            not a_eff.is_cuda or self.source.n * self.target.n >= 4_000_000
+        )
+        if use_gather:
             # Read only the rows of A for source neurons that spiked this step
             # (numerically identical; faster only when few are active).
             active = s.any(dim=0).nonzero(as_tuple=False).squeeze(1)
diff --git a/bindsnet/network/topology_features.py b/bindsnet/network/topology_features.py
index 3cfc5a672..99eeded61 100644
--- a/bindsnet/network/topology_features.py
+++ b/bindsnet/network/topology_features.py
@@ -1093,12 +1093,12 @@ def compute(self, s):
         # language=rst
         """
         Proxy to run a parent feature's side-effect (e.g. normalize/update) from
-        inside the pipeline. Returns ``1`` so it is the identity factor in the fold.
+        inside the pipeline. Returns ``None`` so the fold skips it entirely.
         """
 
         # sub_feature should be defined in the non-abstract constructor
         self.sub_feature()
-        return 1
+        return None
 
 
 class Normalization(AbstractSubFeature):
diff --git a/examples/benchmark/sparse_compute and foldable pipeline/_bench_common.py b/examples/benchmark/sparse_compute and foldable pipeline/_bench_common.py
index 90811f3ee..91c21cffe 100644
--- a/examples/benchmark/sparse_compute and foldable pipeline/_bench_common.py	
+++ b/examples/benchmark/sparse_compute and foldable pipeline/_bench_common.py	
@@ -24,9 +24,10 @@
 import sys
 import time
 
-_HERE = os.path.dirname(os.path.abspath(__file__))
-_ROOT = os.path.dirname(os.path.dirname(_HERE))  # repo root (for ``bindsnet``)
-_STRESS = os.path.join(_ROOT, "stress_test")  # for ``example_network``
+_HERE = os.path.dirname(os.path.abspath(__file__))  # .../examples/benchmark/<name>
+_EXAMPLES = os.path.dirname(os.path.dirname(_HERE))  # .../examples
+_ROOT = os.path.dirname(_EXAMPLES)  # repo root (for ``bindsnet``)
+_STRESS = os.path.join(_EXAMPLES, "stress_test")  # for ``example_network``
 for _p in (_ROOT, _STRESS):
     if _p not in sys.path:
         sys.path.insert(0, _p)
diff --git a/test/network/test_connections.py b/test/network/test_connections.py
index f692f0885..d3f6626ca 100644
--- a/test/network/test_connections.py
+++ b/test/network/test_connections.py
@@ -22,12 +22,7 @@ class TestConnection:
     Tests all stable groups of neurons / nodes.
     """
 
-    def __init__(self):
-        if torch.cuda.is_available():
-            self.device = torch.device("cuda:0")
-        else:
-            self.device = torch.device("cpu:0")
-        print(f"Using device '{self.device}' for the test")
+    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 
     def test_transfer(self):
         if not torch.cuda.is_available():
@@ -48,7 +43,7 @@ def test_transfer(self):
             l_b = LIFNodes(shape=[1, 26, 26])
             connection = conn_type(l_a, l_b, *args, **kwargs)
 
-            connection.to()
+            connection.to(self.device)
 
             connection_tensors = [
                 k
@@ -73,7 +68,9 @@ def test_transfer(self):
                 print(d, d == torch.device("cuda:0"))
                 assert d == torch.device("cuda:0")
 
-    def test_weights(self, conn_type, shape_a, shape_b, shape_w, *args, **kwargs):
+    # Not named test_*: this is a manual matrix check driven from __main__ (it
+    # takes arguments, so pytest cannot collect it).
+    def check_weights(self, conn_type, shape_a, shape_b, shape_w, *args, **kwargs):
         print("Testing:", conn_type)
         time = 100
         weights = [None, torch.Tensor(*shape_w)]
@@ -169,9 +166,7 @@ def test_weights(self, conn_type, shape_a, shape_b, shape_w, *args, **kwargs):
 
 class TestMultiCompartmentConnection:
 
-    def __init__(self):
-        self.device = torch.device("cpu")
-        print(f"Using device '{self.device}' for the MCC test")
+    device = torch.device("cpu")
 
     # ----------------------------------------------------------------------- #
     # Helpers                                                                 #
@@ -477,6 +472,98 @@ def pipes():
                 ).compute(s)
                 assert torch.allclose(dense, sparse, atol=1e-5)
 
+    # ----------------------------------------------------------------------- #
+    # Regressions                                                             #
+    # ----------------------------------------------------------------------- #
+
+    def test_sparse_probability_feature(self):
+        # Probability(sparse=True) stores its value as a sparse [1, src, tgt]
+        # tensor; the fold must densify it to a [src, tgt] factor (regression:
+        # "expand is unsupported for Sparse tensors").
+        s = torch.tensor([[1.0, 0.0, 1.0], [1.0, 1.0, 1.0]])
+        w = torch.tensor([[0.1, 0.2], [0.3, 0.4], [0.5, 0.6]])
+        for sparse_compute in (False, True):
+            passes = self._make_mcc(
+                [
+                    tf.Weight(name="w", value=w.clone()),
+                    tf.Probability(name="p", value=torch.ones(3, 2), sparse=True),
+                ],
+                3,
+                2,
+                batch=2,
+                sparse_compute=sparse_compute,
+            )
+            assert torch.allclose(passes.compute(s), s @ w, atol=1e-6)
+            blocked = self._make_mcc(
+                [
+                    tf.Weight(name="w", value=w.clone()),
+                    tf.Probability(name="p", value=torch.zeros(3, 2), sparse=True),
+                ],
+                3,
+                2,
+                batch=2,
+                sparse_compute=sparse_compute,
+            )
+            assert torch.allclose(blocked.compute(s), torch.zeros(2, 2), atol=1e-6)
+
+    def test_empty_pipeline_fan_in(self):
+        # No features: every source contributes with unit weight.
+        conn = self._make_mcc([], 5, 3, batch=2)
+        s = torch.tensor([[1.0, 1.0, 0.0, 0.0, 1.0], [0.0, 0.0, 0.0, 0.0, 0.0]])
+        out = conn.compute(s)
+        assert torch.allclose(out, s.sum(1, keepdim=True).expand(2, 3))
+
+    def test_subfeature_folds_as_identity(self):
+        # A sub-feature runs its side effect and contributes nothing to the
+        # fold, even when it precedes every real feature (regression: returned
+        # the int 1, which broke torch.is_floating_point when first).
+        w = torch.tensor([[0.5, 1.5], [1.0, 0.5], [0.5, 1.0]])
+        conn = self._make_mcc([tf.Weight(name="w", value=w.clone(), norm=2.0)], 3, 2)
+        wf = conn.pipeline[0]
+        conn.pipeline = [tf.Normalization(name="n", parent_feature=wf), wf]
+        s = torch.tensor([[1.0, 1.0, 1.0]])
+        out = conn.compute(s)
+        # normalize ran inside the fold (before Weight), so each target column
+        # of the weight sums to `norm` and the output uses those values.
+        assert torch.allclose(wf.value.sum(0), torch.full((2,), 2.0), atol=1e-5)
+        assert torch.allclose(out, s @ wf.value, atol=1e-5)
+
+    def test_mstdp_decay_tracks_dt_change(self):
+        # The cached MSTDP trace-decay factors must follow connection.dt
+        # (regression: frozen at first update).
+        w0 = torch.rand(3, 2)
+        conn, feat = self._learning_conn(mcc.MSTDP, w0, nu=(0.1, 0.1))
+        rule = feat.learning_rule
+        conn.source.s = torch.tensor([[1.0, 0.0, 0.0]])
+        conn.target.s = torch.tensor([[0.0, 1.0]])
+        rule.update(reward=0.0)
+        assert torch.allclose(rule._decay_plus, torch.exp(torch.tensor(-1.0 / 20.0)))
+        conn.dt = 5.0
+        rule.update(reward=0.0)
+        assert torch.allclose(rule._decay_plus, torch.exp(torch.tensor(-5.0 / 20.0)))
+
+    def test_sparse_compute_matches_dense_cuda(self):
+        # On CUDA the gather is gated by connection size; both the gated-off
+        # (small) and gated-on (large) paths must match the dense result.
+        if not torch.cuda.is_available():
+            return
+        torch.manual_seed(2)
+        dev = torch.device("cuda")
+        for src_n, tgt_n in ((80, 40), (2100, 2000)):  # below / above the gate
+            w = torch.randn(src_n, tgt_n, device=dev)
+            s = (torch.rand(2, src_n, device=dev) > 0.9).float()
+            outs = []
+            for sc in (False, True):
+                conn = MulticompartmentConnection(
+                    source=Input(n=src_n),
+                    target=LIFNodes(n=tgt_n),
+                    device=dev,
+                    pipeline=[tf.Weight(name="w", value=w.clone())],
+                    sparse_compute=sc,
+                )
+                outs.append(conn.compute(s))
+            assert torch.allclose(outs[0], outs[1], atol=1e-4)
+
     # ----------------------------------------------------------------------- #
     # MCC learning rules                                                      #
     # ----------------------------------------------------------------------- #
@@ -562,25 +649,11 @@ def test_hebbian_predictable(self):
 
 
 if __name__ == "__main__":
-    # MulticompartmentConnection + MCC learning-rule tests.
+    # MulticompartmentConnection + MCC learning-rule tests (discovered
+    # dynamically so this list cannot go stale).
     mcc_tester = TestMultiCompartmentConnection()
     mcc_tests = [
-        mcc_tester.test_weight_feature_output,
-        mcc_tester.test_mask_feature_output,
-        mcc_tester.test_bias_feature_output,
-        mcc_tester.test_intensity_feature_output,
-        mcc_tester.test_degradation_feature_output,
-        mcc_tester.test_probability_feature_deterministic_bounds,
-        mcc_tester.test_meanfield_not_implemented,
-        mcc_tester.test_adaptation_features_output,
-        mcc_tester.test_pipeline_matches_expansion,
-        mcc_tester.test_sparse_compute_matches_dense,
-        mcc_tester.test_noop_leaves_weight_unchanged,
-        mcc_tester.test_postpre_predictable,
-        mcc_tester.test_learning_respects_range_clamp,
-        mcc_tester.test_mstdp_predictable,
-        mcc_tester.test_mstdpet_predictable,
-        mcc_tester.test_hebbian_predictable,
+        getattr(mcc_tester, n) for n in sorted(dir(mcc_tester)) if n.startswith("test_")
     ]
     for mcc_test in mcc_tests:
         mcc_test()
@@ -602,7 +675,7 @@ def test_hebbian_predictable(self):
     for update_rule in (Hebbian, PostPre, WeightDependentPostPre, MSTDP, MSTDPET, Rmax):
         print("Learning Rule:", update_rule)
         for conn_type, arg in zip(conn_types, args):
-            tester.test_weights(conn_type, nu=1e-2, update_rule=update_rule, *arg)
+            tester.check_weights(conn_type, nu=1e-2, update_rule=update_rule, *arg)
 
     # Other connections
     # Note: Does not include MaxPool2dConnection because this connection
@@ -610,4 +683,4 @@ def test_hebbian_predictable(self):
     conn_types = [MeanFieldConnection]
     args = [[[1, 28, 28], [1, 26, 26], (1, 26), 3, 1]]
     for conn_type, arg in zip(conn_types, args):
-        tester.test_weights(conn_type, decay=1, update_rule=NoOp, *arg)
+        tester.check_weights(conn_type, decay=1, update_rule=NoOp, *arg)

From e3081237f6d34f3c8b25a436c6ce3f35d62eaf96 Mon Sep 17 00:00:00 2001
From: Hananel Hazan <Hananel@Hazan.org.il>
Date: Mon, 3 Aug 2026 21:01:52 -0400
Subject: [PATCH 11/14] Performance: fold cache, rank-1 MSTDP, cached MSTDPET
 decays

- topology.py: cache the folded (a_eff, b_sum) whenever every pipeline
  feature is static (is_static flag; Probability, MeanField, adaptation
  features, and sub-features are dynamic). Invalidated on learning
  updates, normalize, reset, and device/dtype moves. Weight*Mask*
  Intensity at 100x20000: 4506 -> 66 us/step CPU, 321 -> 58 us GPU.
- topology_features.py: Weight's per-time-step normalization is deferred
  by the connection until after the fold consumes the pre-norm value,
  removing the full-matrix clone every step. Fix the reduction assertion
  (isinstance(x, callable) raised TypeError for any custom reduction).
- MCC_learning.py: MSTDP with the default batch reductions applies the
  reward-weighted eligibility as two rank-1 addmm_ updates instead of
  materializing [batch, src, tgt] every step (5077 -> 899 us CPU,
  834 -> 332 us GPU at 100x20000; exact same numbers, verified to 1e-7).
  Dense path kept for averaging buffers, custom reductions, and sparse
  weights. Skip the decay multiply when decay is the no-op 1.0. Give
  MSTDPET the same (dt, device)-keyed decay/learning-rate cache as MSTDP
  (was 3 exp() + 2 host-to-device copies per step).
- New tests: rank-1 MSTDP == dense eligibility (batch 1/4, tensor
  rewards), fold-cache storage + invalidation, deferred time-step norm.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 bindsnet/datasets/torchvision_wrapper.py |   9 +-
 bindsnet/learning/MCC_learning.py        | 138 +++++++++++++++--------
 bindsnet/network/topology.py             |  95 ++++++++++------
 bindsnet/network/topology_features.py    |  41 +++++--
 test/network/test_connections.py         | 120 ++++++++++++++++++++
 5 files changed, 311 insertions(+), 92 deletions(-)

diff --git a/bindsnet/datasets/torchvision_wrapper.py b/bindsnet/datasets/torchvision_wrapper.py
index 23960a04f..b1e36519e 100644
--- a/bindsnet/datasets/torchvision_wrapper.py
+++ b/bindsnet/datasets/torchvision_wrapper.py
@@ -16,18 +16,13 @@ def create_torchvision_dataset_wrapper(ds_type):
         ds_type = getattr(torchDB, ds_type)
 
     class TorchvisionDatasetWrapper(ds_type):
-        __doc__ = (
-            """BindsNET torchvision dataset wrapper for:
+        __doc__ = """BindsNET torchvision dataset wrapper for:
 
         The core difference is the output of __getitem__ is no longer
         (image, label) rather a dictionary containing the image, label,
         and their encoded versions if encoders were provided.
 
-            \n\n"""
-            + str(ds_type)
-            if ds_type.__doc__ is None
-            else ds_type.__doc__
-        )
+            \n\n""" + str(ds_type) if ds_type.__doc__ is None else ds_type.__doc__
 
         def __init__(
             self,
diff --git a/bindsnet/learning/MCC_learning.py b/bindsnet/learning/MCC_learning.py
index fd26dca15..be067e31f 100644
--- a/bindsnet/learning/MCC_learning.py
+++ b/bindsnet/learning/MCC_learning.py
@@ -89,8 +89,9 @@ def update(self, **kwargs) -> None:
         Abstract method for a learning rule update.
         """
 
-        # Implement decay.
-        if self.decay:
+        # Implement decay (self.decay == 1.0 is the no-op default; skip the
+        # full-matrix multiply in that case).
+        if self.decay != 1.0:
             self.feature_value *= self.decay
 
         # Enforce polarities
@@ -497,11 +498,6 @@ def _connection_update(self, **kwargs) -> None:
                 self.target.n,
                 device=self.target.s.device,
             )
-        if not hasattr(self, "eligibility"):
-            self.eligibility = torch.zeros(
-                batch_size, *self.feature_value.shape, device=self.feature_value.device
-            )
-
         # Reshape pre- and post-synaptic spikes.
         source_s = self.source.s.view(batch_size, -1).float()
         target_s = self.target.s.view(batch_size, -1).float()
@@ -532,27 +528,66 @@ def _connection_update(self, **kwargs) -> None:
             else torch.as_tensor(a_minus, device=self.feature_value.device)
         )
 
-        # Compute weight update based on the eligibility value of the past timestep.
-        update = reward * self.eligibility
+        # With no averaging and the standard batch reductions (squeeze for
+        # batch size 1, sum otherwise), the [batch, src, tgt] eligibility never
+        # needs to be materialized: its batch reduction is a sum of two matrix
+        # products applied to the weights directly. The eligibility of the past
+        # timestep is outer(p_plus, target_s) + outer(source_s, p_minus), with
+        # p_plus/p_minus still holding their previous-step values here.
+        fast = (
+            self.average_update == 0
+            and self.reduction in (torch.squeeze, torch.sum)
+            and not self.feature_value.is_sparse
+        )
+        if fast:
+            if hasattr(self, "_prev_target_s"):
+                if isinstance(reward, torch.Tensor):
+                    # Keep reward on-device (no host sync for tensor rewards).
+                    update = (
+                        self.p_plus.t() @ self._prev_target_s
+                        + self._prev_source_s.t() @ self.p_minus
+                    )
+                    self.feature_value += (self.nu[0].to(dev) * reward) * update
+                else:
+                    alpha = float(self.nu[0]) * reward
+                    if alpha != 0.0:
+                        self.feature_value.addmm_(
+                            self.p_plus.t(), self._prev_target_s, alpha=alpha
+                        )
+                        self.feature_value.addmm_(
+                            self._prev_source_s.t(), self.p_minus, alpha=alpha
+                        )
+        else:
+            # Dense-eligibility path: averaging buffers, custom reductions, or
+            # sparse weights.
+            if not hasattr(self, "eligibility"):
+                self.eligibility = torch.zeros(
+                    batch_size,
+                    *self.feature_value.shape,
+                    device=self.feature_value.device,
+                )
 
-        if self.average_update > 0:
-            self.average_buffer[self.average_buffer_index] = self.reduction(
-                update, dim=0
-            )
-            self.average_buffer_index = (
-                self.average_buffer_index + 1
-            ) % self.average_update
+            # Compute weight update based on the eligibility value of the past timestep.
+            update = reward * self.eligibility
 
-            if self.continues_update or self.average_buffer_index == 0:
-                update = self.nu[0] * torch.mean(self.average_buffer, dim=0)
+            if self.average_update > 0:
+                self.average_buffer[self.average_buffer_index] = self.reduction(
+                    update, dim=0
+                )
+                self.average_buffer_index = (
+                    self.average_buffer_index + 1
+                ) % self.average_update
+
+                if self.continues_update or self.average_buffer_index == 0:
+                    update = self.nu[0] * torch.mean(self.average_buffer, dim=0)
+                    if self.feature_value.is_sparse:
+                        update = update.to_sparse()
+                    self.feature_value += update
+            else:
+                update = self.nu[0] * self.reduction(update, dim=0)
                 if self.feature_value.is_sparse:
                     update = update.to_sparse()
                 self.feature_value += update
-        else:
-            update = self.nu[0] * self.reduction(update, dim=0)
-            if self.feature_value.is_sparse:
-                update = update.to_sparse()
-            self.feature_value += update
 
         # Update P^+ and P^- values.
         self.p_plus *= self._decay_plus
@@ -560,10 +595,15 @@ def _connection_update(self, **kwargs) -> None:
         self.p_minus *= self._decay_minus
         self.p_minus += a_minus * target_s
 
-        # Calculate point eligibility value.
-        self.eligibility = torch.bmm(
-            self.p_plus.unsqueeze(2), target_s.unsqueeze(1)
-        ) + torch.bmm(source_s.unsqueeze(2), self.p_minus.unsqueeze(1))
+        if fast:
+            # Keep this step's spikes for the next step's rank-1 update.
+            self._prev_source_s = source_s.clone()
+            self._prev_target_s = target_s.clone()
+        else:
+            # Calculate point eligibility value.
+            self.eligibility = torch.bmm(
+                self.p_plus.unsqueeze(2), target_s.unsqueeze(1)
+            ) + torch.bmm(source_s.unsqueeze(2), self.p_minus.unsqueeze(1))
 
         super().update()
 
@@ -688,26 +728,36 @@ def _connection_update(self, **kwargs) -> None:
 
         # Parse keyword arguments.
         reward = kwargs["reward"]
-        a_plus = kwargs.get("a_plus", 1.0)
-        # if isinstance(a_plus, dict):
-        #     for k, v in a_plus.items():
-        #         a_plus[k] = torch.tensor(v, device=self.feature_value.device)
-        # else:
-        a_plus = torch.tensor(a_plus, device=self.feature_value.device)
-        a_minus = kwargs.get("a_minus", -1.0)
-        # if isinstance(a_minus, dict):
-        #     for k, v in a_minus.items():
-        #         a_minus[k] = torch.tensor(v, device=self.feature_value.device)
-        # else:
-        a_minus = torch.tensor(a_minus, device=self.feature_value.device)
+
+        # Build learning-rate and decay tensors, cached on (dt, device) so a
+        # change to either (network.dt edits, .to(device) moves) recomputes them.
+        dev = self.feature_value.device
+        dt = float(self.connection.dt)
+        if getattr(self, "_decay_key", None) != (dt, dev):
+            self._decay_key = (dt, dev)
+            self._a_plus_default = torch.tensor(1.0, device=dev)
+            self._a_minus_default = torch.tensor(-1.0, device=dev)
+            self._decay_plus = torch.exp(-dt / self.tc_plus.to(dev))
+            self._decay_minus = torch.exp(-dt / self.tc_minus.to(dev))
+            self._decay_e_trace = torch.exp(-dt / self.tc_e_trace.to(dev))
+        a_plus = kwargs.get("a_plus", None)
+        a_plus = (
+            self._a_plus_default
+            if a_plus is None
+            else torch.as_tensor(a_plus, device=dev)
+        )
+        a_minus = kwargs.get("a_minus", None)
+        a_minus = (
+            self._a_minus_default
+            if a_minus is None
+            else torch.as_tensor(a_minus, device=dev)
+        )
 
         # Calculate value of eligibility trace based on the value
         # of the point eligibility value of the past timestep.
         # Note: eligibility = [source.n, target.n] > 0 where source and target spiked
         # Note: high negs. ->
-        self.eligibility_trace *= torch.exp(
-            -self.connection.dt / self.tc_e_trace
-        )  # Decay
+        self.eligibility_trace *= self._decay_e_trace  # Decay
         self.eligibility_trace += self.eligibility / self.tc_e_trace  # Additive changes
         # ^ Also effected by delay in last step
 
@@ -733,9 +783,9 @@ def _connection_update(self, **kwargs) -> None:
             self.feature_value += update
 
         # Update P^+ and P^- values.
-        self.p_plus *= torch.exp(-self.connection.dt / self.tc_plus)  # Decay
+        self.p_plus *= self._decay_plus  # Decay
         self.p_plus += a_plus * source_s  # Scaled source spikes
-        self.p_minus *= torch.exp(-self.connection.dt / self.tc_minus)  # Decay
+        self.p_minus *= self._decay_minus  # Decay
         self.p_minus += a_minus * target_s  # Scaled target spikes
 
         # Notes:
diff --git a/bindsnet/network/topology.py b/bindsnet/network/topology.py
index f64227f26..b9b8a9702 100644
--- a/bindsnet/network/topology.py
+++ b/bindsnet/network/topology.py
@@ -442,6 +442,15 @@ def __init__(
 
         self.sparse_compute = sparse_compute
 
+        # Cached (a_eff, b_sum) for pipelines whose features are all static
+        # (see AbstractFeature.is_static). Invalidated whenever a feature can
+        # change: learning updates, normalize, reset, device/dtype moves.
+        self._fold_cache = None
+
+    def _apply(self, fn, recurse=True):
+        self._fold_cache = None
+        return super()._apply(fn, recurse)
+
     def compute(self, s: torch.Tensor) -> torch.Tensor:
         # language=rst
         """
@@ -463,34 +472,50 @@ def compute(self, s: torch.Tensor) -> torch.Tensor:
         """
         s = s.view(s.size(0), self.source.n)
 
-        # running product of multiplicative factors, [source.n, target.n]
-        a_eff = None
-        # running additive offset, [source.n, target.n]; None while still zero
-        b_eff = None
-        for f in self.pipeline:
-            factor = f.compute(s)
-            if factor is None:
-                # Side-effect-only pipeline entries (sub-features) fold as identity.
-                continue
-            if isinstance(factor, torch.Tensor) and factor.is_sparse:
-                # Sparse feature values carry a leading batch dim ([1, src, tgt])
-                # from prime_feature; densify to the fold's [src, tgt] shape.
-                factor = factor.to_dense().view(self.source.n, self.target.n)
-            op = getattr(f, "op", "mul")
-            if op == "mul":
-                a_eff = factor if a_eff is None else a_eff * factor
-                if b_eff is not None:
-                    b_eff = b_eff * factor
-            else:  # additive contribution: "add" -> +factor, "sub" -> -factor
-                term = factor if op == "add" else -factor
-                b_eff = term if b_eff is None else b_eff + term
-
-        if a_eff is None:
-            # Degenerate pipeline with no multiplicative feature: every source
-            # neuron contributes with unit weight.
-            a_eff = torch.ones(self.source.n, self.target.n, device=s.device)
-        if not torch.is_floating_point(a_eff):
-            a_eff = a_eff.float()
+        deferred = None
+        if self._fold_cache is not None:
+            a_eff, b_sum = self._fold_cache
+        else:
+            # running product of multiplicative factors, [source.n, target.n]
+            a_eff = None
+            # running additive offset, [source.n, target.n]; None while still zero
+            b_eff = None
+            for f in self.pipeline:
+                factor = f.compute(s)
+                # Compute-time side effects (e.g. per-time-step weight
+                # normalization) run after the fold has consumed this value.
+                d = getattr(f, "defer", None)
+                if d is not None:
+                    deferred = [d] if deferred is None else deferred + [d]
+                if factor is None:
+                    # Side-effect-only pipeline entries (sub-features) fold as identity.
+                    continue
+                if isinstance(factor, torch.Tensor) and factor.is_sparse:
+                    # Sparse feature values carry a leading batch dim ([1, src, tgt])
+                    # from prime_feature; densify to the fold's [src, tgt] shape.
+                    factor = factor.to_dense().view(self.source.n, self.target.n)
+                op = getattr(f, "op", "mul")
+                if op == "mul":
+                    a_eff = factor if a_eff is None else a_eff * factor
+                    if b_eff is not None:
+                        b_eff = b_eff * factor
+                else:  # additive contribution: "add" -> +factor, "sub" -> -factor
+                    term = factor if op == "add" else -factor
+                    b_eff = term if b_eff is None else b_eff + term
+
+            if a_eff is None:
+                # Degenerate pipeline with no multiplicative feature: every source
+                # neuron contributes with unit weight.
+                a_eff = torch.ones(self.source.n, self.target.n, device=s.device)
+            if not torch.is_floating_point(a_eff):
+                a_eff = a_eff.float()
+
+            # Additive terms apply to every synapse regardless of spikes, so
+            # their contribution is the source-sum, a constant [target.n] row.
+            b_sum = b_eff.sum(dim=0) if b_eff is not None else None
+
+            if all(getattr(f, "is_static", True) for f in self.pipeline):
+                self._fold_cache = (a_eff, b_sum)
 
         # The gather pays for its ``nonzero()`` only where that sync is expensive
         # relative to the matmul: on CUDA it needs a large weight matrix to win;
@@ -511,11 +536,12 @@ def compute(self, s: torch.Tensor) -> torch.Tensor:
         else:
             out = s.to(a_eff.dtype) @ a_eff
 
-        # Additive terms apply to every synapse regardless of spikes, so sum over
-        # all source rows (the closed form of the [batch, source.n, target.n]
-        # source-sum).
-        if b_eff is not None:
-            out = out + b_eff.sum(dim=0)
+        if b_sum is not None:
+            out = out + b_sum
+
+        if deferred is not None:
+            for fn in deferred:
+                fn()
 
         if self.traces:
             self.activity = out
@@ -559,6 +585,7 @@ def update(self, **kwargs) -> None:
         learning = kwargs.get("learning", False)
         if learning and not self.manual_update:
             # Pipeline learning
+            self._fold_cache = None
             for f in self.pipeline:
                 f.update(**kwargs)
 
@@ -568,6 +595,7 @@ def normalize(self) -> None:
         Normalize all features in the connection.
         """
         # Normalize pipeline features
+        self._fold_cache = None
         for f in self.pipeline:
             f.normalize()
 
@@ -578,6 +606,7 @@ def reset_state_variables(self) -> None:
         """
         super().reset_state_variables()
 
+        self._fold_cache = None
         for f in self.pipeline:
             f.reset_state_variables()
 
diff --git a/bindsnet/network/topology_features.py b/bindsnet/network/topology_features.py
index 99eeded61..7abcd6368 100644
--- a/bindsnet/network/topology_features.py
+++ b/bindsnet/network/topology_features.py
@@ -24,6 +24,13 @@ class AbstractFeature(ABC):
     # returned by :meth:`compute`).
     op = "mul"
 
+    # Whether :meth:`compute` returns the same value every step until the
+    # feature is mutated through the connection (learning update, normalize,
+    # reset, device move). Static-only pipelines let the connection cache the
+    # folded factors between mutations. Features whose value depends on the
+    # incoming spikes or is resampled per step must set this to ``False``.
+    is_static = True
+
     @abstractmethod
     def __init__(
         self,
@@ -122,9 +129,9 @@ def __init__(
         ), "Feature {0}'s nu should be of type list or tuple, not {1}".format(
             name, type(nu)
         )
-        assert reduction is None or isinstance(
-            reduction, callable
-        ), "Feature {0}'s reduction should be of type callable, not {1}".format(
+        assert reduction is None or callable(
+            reduction
+        ), "Feature {0}'s reduction should be callable, not {1}".format(
             name, type(reduction)
         )
         assert decay is None or isinstance(
@@ -433,6 +440,9 @@ def sparse_bernoulli(self):
         non_zero = values[mask]
         return torch.sparse_coo_tensor(indices, non_zero, self.value.size())
 
+    # Resampled every step; never cacheable.
+    is_static = False
+
     def compute(self, s) -> Union[torch.Tensor, float, int]:
         # Factor: a fresh Bernoulli draw each step (resampled, never cached).
         if self.sparse:
@@ -561,6 +571,9 @@ def prime_feature(self, connection, device, **kwargs) -> None:
 
 
 class MeanField(AbstractFeature):
+    # Depends on the incoming spikes; never cacheable.
+    is_static = False
+
     def __init__(self) -> None:
         # language=rst
         """
@@ -624,6 +637,12 @@ def __init__(
 
         self.norm_frequency = norm_frequency
         self.enforce_polarity = enforce_polarity
+        if norm_frequency == "time step":
+            # Normalization mutates ``value`` on every compute; the connection
+            # runs ``defer`` after the fold has consumed the pre-normalization
+            # value (this avoids cloning the full matrix every step).
+            self.is_static = False
+            self.defer = lambda: self.normalize(time_step_norm=True)
         super().__init__(
             name=name,
             value=value,
@@ -649,11 +668,7 @@ def compute(self, s) -> Union[torch.Tensor, float, int]:
             self.value[~pos_mask] = 0.0001
             self.value[~neg_mask] = -0.0001
 
-        factor = self.value
-        if self.norm_frequency == "time step":
-            factor = factor.clone()
-            self.normalize(time_step_norm=True)
-        return factor
+        return self.value
 
     def prime_feature(self, connection, device, **kwargs) -> None:
         #### Initialize value ####
@@ -835,6 +850,9 @@ def compute(self, s) -> Union[torch.Tensor, float, int]:
 
 
 class AdaptationBaseSynapsHistory(AbstractFeature):
+    # Value evolves with the spike history every step; never cacheable.
+    is_static = False
+
     def __init__(
         self,
         name: str,
@@ -949,6 +967,9 @@ def reset_state_variables(
 
 
 class AdaptationBaseOtherSynaps(AbstractFeature):
+    # Value evolves with the spike history every step; never cacheable.
+    is_static = False
+
     def __init__(
         self,
         name: str,
@@ -1072,6 +1093,10 @@ class AbstractSubFeature(ABC):
     execution.
     """
 
+    # Runs a side effect on every step; the pipeline must never be cached
+    # around it.
+    is_static = False
+
     @abstractmethod
     def __init__(
         self,
diff --git a/test/network/test_connections.py b/test/network/test_connections.py
index d3f6626ca..fec421da2 100644
--- a/test/network/test_connections.py
+++ b/test/network/test_connections.py
@@ -564,6 +564,126 @@ def test_sparse_compute_matches_dense_cuda(self):
                 outs.append(conn.compute(s))
             assert torch.allclose(outs[0], outs[1], atol=1e-4)
 
+    # ----------------------------------------------------------------------- #
+    # Performance-path equivalence                                            #
+    # ----------------------------------------------------------------------- #
+
+    def _run_mstdp(self, batch, reduction, reward_seq, seed=0):
+        """Run an MSTDP-learned Weight over a spike/reward sequence."""
+        torch.manual_seed(seed)
+        src_n, tgt_n = 7, 5
+        w0 = torch.rand(src_n, tgt_n)
+        conn = self._make_mcc(
+            [
+                tf.Weight(
+                    name="w",
+                    value=w0.clone(),
+                    learning_rule=mcc.MSTDP,
+                    nu=(0.05, 0.05),
+                    range=[-10, 10],
+                    reduction=reduction,
+                )
+            ],
+            src_n,
+            tgt_n,
+            batch=batch,
+        )
+        feat = conn.pipeline[0]
+        rule = feat.learning_rule
+        torch.manual_seed(seed + 1)
+        for r in reward_seq:
+            conn.source.s = torch.bernoulli(torch.full((batch, src_n), 0.4))
+            conn.target.s = torch.bernoulli(torch.full((batch, tgt_n), 0.4))
+            rule.update(reward=r)
+        return feat.value.clone()
+
+    def test_mstdp_rank1_matches_dense(self):
+        # The rank-1 addmm_ fast path (default reductions) must match the
+        # dense-eligibility path (forced here via equivalent custom lambdas).
+        rewards = [0.0, 1.0, 0.5, -2.0, 1.0, 0.0, 3.0]
+        slow_squeeze = lambda x, dim: torch.squeeze(x, dim)
+        slow_sum = lambda x, dim: torch.sum(x, dim)
+        for batch, fast_red, slow_red in (
+            (1, None, slow_squeeze),
+            (4, torch.sum, slow_sum),
+        ):
+            w_fast = self._run_mstdp(batch, fast_red, rewards)
+            w_slow = self._run_mstdp(batch, slow_red, rewards)
+            assert torch.allclose(
+                w_fast, w_slow, atol=1e-5
+            ), f"batch={batch}: {(w_fast - w_slow).abs().max()}"
+        # Tensor rewards take the sync-free tensor branch; same numbers.
+        w_fast = self._run_mstdp(1, None, [torch.tensor(r) for r in rewards])
+        w_slow = self._run_mstdp(1, slow_squeeze, rewards)
+        assert torch.allclose(w_fast, w_slow, atol=1e-5)
+
+    def test_fold_cache_static_and_invalidation(self):
+        # Static pipelines cache the folded factors; dynamic ones must not;
+        # learning updates through the connection invalidate the cache.
+        s = torch.ones(1, 6, dtype=torch.bool)
+        w = torch.rand(6, 4)
+        static = self._make_mcc([tf.Weight(name="w", value=w.clone())], 6, 4)
+        out1 = static.compute(s)
+        assert static._fold_cache is not None
+        assert torch.allclose(static.compute(s), out1)
+
+        dynamic = self._make_mcc(
+            [
+                tf.Weight(name="w", value=w.clone()),
+                tf.Probability(name="p", value=torch.full((6, 4), 0.5)),
+            ],
+            6,
+            4,
+        )
+        dynamic.compute(s)
+        assert dynamic._fold_cache is None
+
+        # A learning step through connection.update must drop the cache and
+        # the next compute must see the new weights.
+        learned = self._make_mcc(
+            [
+                tf.Weight(
+                    name="w",
+                    value=w.clone(),
+                    learning_rule=mcc.PostPre,
+                    nu=(0.5, 0.5),
+                    range=[-10, 10],
+                )
+            ],
+            6,
+            4,
+        )
+        learned.compute(s)
+        learned.source.s = torch.ones(1, 6)
+        learned.target.s = torch.ones(1, 4)
+        learned.source.x = torch.ones(1, 6)
+        learned.target.x = torch.ones(1, 4)
+        learned.update(learning=True)
+        assert learned._fold_cache is None
+        w_new = learned.pipeline[0].value
+        assert torch.allclose(learned.compute(s), s.float() @ w_new, atol=1e-5)
+
+    def test_time_step_norm_deferred(self):
+        # Per-time-step normalization runs after the fold: each step's output
+        # uses the pre-normalization weights (old expansion semantics).
+        w0 = torch.tensor([[1.0, 4.0], [3.0, 4.0], [1.0, 2.0]])
+        conn = self._make_mcc(
+            [
+                tf.Weight(
+                    name="w", value=w0.clone(), norm=1.0, norm_frequency="time step"
+                )
+            ],
+            3,
+            2,
+        )
+        s = torch.ones(1, 3)
+        ref_w = w0.clone()
+        for step in range(3):
+            out = conn.compute(s)
+            assert torch.allclose(out, s @ ref_w, atol=1e-5), f"step {step}"
+            ref_w = ref_w / ref_w.abs().sum(0, keepdim=True)
+        assert conn._fold_cache is None  # never cached while norm runs per step
+
     # ----------------------------------------------------------------------- #
     # MCC learning rules                                                      #
     # ----------------------------------------------------------------------- #

From 951e2e07977f765f7b34200c5220b3b271355bb2 Mon Sep 17 00:00:00 2001
From: Hananel Hazan <Hananel@Hazan.org.il>
Date: Mon, 3 Aug 2026 21:08:49 -0400
Subject: [PATCH 12/14] Refactor docstring formatting in
 TorchvisionDatasetWrapper for improved readability

---
 bindsnet/datasets/torchvision_wrapper.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/bindsnet/datasets/torchvision_wrapper.py b/bindsnet/datasets/torchvision_wrapper.py
index b1e36519e..23960a04f 100644
--- a/bindsnet/datasets/torchvision_wrapper.py
+++ b/bindsnet/datasets/torchvision_wrapper.py
@@ -16,13 +16,18 @@ def create_torchvision_dataset_wrapper(ds_type):
         ds_type = getattr(torchDB, ds_type)
 
     class TorchvisionDatasetWrapper(ds_type):
-        __doc__ = """BindsNET torchvision dataset wrapper for:
+        __doc__ = (
+            """BindsNET torchvision dataset wrapper for:
 
         The core difference is the output of __getitem__ is no longer
         (image, label) rather a dictionary containing the image, label,
         and their encoded versions if encoders were provided.
 
-            \n\n""" + str(ds_type) if ds_type.__doc__ is None else ds_type.__doc__
+            \n\n"""
+            + str(ds_type)
+            if ds_type.__doc__ is None
+            else ds_type.__doc__
+        )
 
         def __init__(
             self,

From ddd51630cb4f5a933983e71f6843ddc8932433ee Mon Sep 17 00:00:00 2001
From: Hananel Hazan <Hananel@Hazan.org.il>
Date: Mon, 3 Aug 2026 21:11:09 -0400
Subject: [PATCH 13/14] Improve docstring formatting in
 TorchvisionDatasetWrapper for clarity

---
 .claude/skills/slurm-harvest/harvest_slurm.py | 1 +
 bindsnet/datasets/torchvision_wrapper.py      | 9 ++-------
 2 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/.claude/skills/slurm-harvest/harvest_slurm.py b/.claude/skills/slurm-harvest/harvest_slurm.py
index 564162f17..f07cb404c 100644
--- a/.claude/skills/slurm-harvest/harvest_slurm.py
+++ b/.claude/skills/slurm-harvest/harvest_slurm.py
@@ -14,6 +14,7 @@
     python harvest_slurm.py --label "ES run (E009)" --run-id 2026-06-03-r01 \
         --glob 'scripts/stage_b/slurm_logs/*.out' [--json out.json] [--md]
 """
+
 from __future__ import annotations
 
 import argparse
diff --git a/bindsnet/datasets/torchvision_wrapper.py b/bindsnet/datasets/torchvision_wrapper.py
index 23960a04f..b1e36519e 100644
--- a/bindsnet/datasets/torchvision_wrapper.py
+++ b/bindsnet/datasets/torchvision_wrapper.py
@@ -16,18 +16,13 @@ def create_torchvision_dataset_wrapper(ds_type):
         ds_type = getattr(torchDB, ds_type)
 
     class TorchvisionDatasetWrapper(ds_type):
-        __doc__ = (
-            """BindsNET torchvision dataset wrapper for:
+        __doc__ = """BindsNET torchvision dataset wrapper for:
 
         The core difference is the output of __getitem__ is no longer
         (image, label) rather a dictionary containing the image, label,
         and their encoded versions if encoders were provided.
 
-            \n\n"""
-            + str(ds_type)
-            if ds_type.__doc__ is None
-            else ds_type.__doc__
-        )
+            \n\n""" + str(ds_type) if ds_type.__doc__ is None else ds_type.__doc__
 
         def __init__(
             self,

From 5215a461bf60f33931232957bc0d8f5494f586a4 Mon Sep 17 00:00:00 2001
From: Hananel Hazan <Hananel@Hazan.org.il>
Date: Mon, 3 Aug 2026 21:19:49 -0400
Subject: [PATCH 14/14] fixing

---
 .claude/settings.json                         |   8 -
 .claude/skills/archive-log/SKILL.md           | 100 ----
 .claude/skills/archive-search/SKILL.md        |  95 ----
 .claude/skills/generate-leaderboard/SKILL.md  |  87 ----
 .claude/skills/generate-report/SKILL.md       | 120 -----
 .claude/skills/hpc-submit/SKILL.md            | 256 ----------
 .claude/skills/intake-results/SKILL.md        | 101 ----
 .claude/skills/log-entry/SKILL.md             | 205 --------
 .claude/skills/pax-hpc/SKILL.md               | 292 -----------
 .claude/skills/pax-hpc/references/intake.md   |  92 ----
 .../skills/pax-hpc/references/new-cluster.md  | 125 -----
 .../skills/pax-hpc/references/old-cluster.md  | 105 ----
 .../skills/pax-hpc/references/resilience.md   | 457 ------------------
 .claude/skills/report/SKILL.md                |  53 --
 .claude/skills/slurm-harvest/SKILL.md         |  38 --
 .claude/skills/slurm-harvest/harvest_slurm.py | 190 --------
 .gitignore                                    |   4 +-
 17 files changed, 3 insertions(+), 2325 deletions(-)
 delete mode 100644 .claude/settings.json
 delete mode 100644 .claude/skills/archive-log/SKILL.md
 delete mode 100644 .claude/skills/archive-search/SKILL.md
 delete mode 100644 .claude/skills/generate-leaderboard/SKILL.md
 delete mode 100644 .claude/skills/generate-report/SKILL.md
 delete mode 100644 .claude/skills/hpc-submit/SKILL.md
 delete mode 100644 .claude/skills/intake-results/SKILL.md
 delete mode 100644 .claude/skills/log-entry/SKILL.md
 delete mode 100644 .claude/skills/pax-hpc/SKILL.md
 delete mode 100644 .claude/skills/pax-hpc/references/intake.md
 delete mode 100644 .claude/skills/pax-hpc/references/new-cluster.md
 delete mode 100644 .claude/skills/pax-hpc/references/old-cluster.md
 delete mode 100644 .claude/skills/pax-hpc/references/resilience.md
 delete mode 100644 .claude/skills/report/SKILL.md
 delete mode 100644 .claude/skills/slurm-harvest/SKILL.md
 delete mode 100644 .claude/skills/slurm-harvest/harvest_slurm.py

diff --git a/.claude/settings.json b/.claude/settings.json
deleted file mode 100644
index 3050c4333..000000000
--- a/.claude/settings.json
+++ /dev/null
@@ -1,8 +0,0 @@
-{
-  "permissions": {
-    "allow": [
-      "Bash(curl -sL \"https://zenodo.org/api/records/20695116\")",
-      "Bash(python -c \"import sys,json; d=json.load\\(sys.stdin\\); print\\('doi:', d.get\\('doi'\\)\\); print\\('conceptdoi:', d.get\\('conceptdoi'\\)\\); print\\('title:', d.get\\('metadata',{}\\).get\\('title'\\)\\); print\\('conceptrecid:', d.get\\('conceptrecid'\\)\\)\")"
-    ]
-  }
-}
diff --git a/.claude/skills/archive-log/SKILL.md b/.claude/skills/archive-log/SKILL.md
deleted file mode 100644
index a1416230c..000000000
--- a/.claude/skills/archive-log/SKILL.md
+++ /dev/null
@@ -1,100 +0,0 @@
----
-name: archive-log
-description: Archives old entries from a per-task log file when it grows too large. Moves entries beyond the keep-last threshold to logs/archive/. Also use when the user says "archive the log", "trim the log", or "the log is too long".
-allowed-tools: Bash, Read, Edit
----
-
-# Archive Old Log Entries
-
-## When to invoke
-
-- Manually, when a task log is getting unwieldy
-- Automatically, called by `/log-entry` when entry count > 10
-- Automatically, called by `/intake-results` Step 8
-- During migration from the old monolithic LOG.md
-
-## Step 1 -- Check the current state
-
-```bash
-# Count entries
-grep -c "^## " logs/LOG.<task>.md
-
-# See the oldest entries that will be archived
-grep "^## " logs/LOG.<task>.md | tail -10
-```
-
-## Step 2 -- Dry-run the archive script
-
-```bash
-python scripts/archive_log.py --task <task> --keep-last 5 --dry-run
-```
-
-Read the output carefully:
-- How many entries will be archived?
-- What is the archive filename?
-- What pointer line will be written into the active log?
-
-## Step 3 -- Execute
-
-```bash
-python scripts/archive_log.py --task <task> --keep-last 5
-```
-
-## Step 4 -- Verify
-
-```bash
-# Active log should now have 5 entries + 1 pointer line near the bottom
-grep -c "^## " logs/LOG.<task>.md
-grep "^> Archived entries" logs/LOG.<task>.md
-
-# Archive file should exist
-ls logs/archive/LOG.<task>.*.md
-```
-
-### Pinned block check
-
-```bash
-head -5 logs/LOG.<task>.md | grep "📌"
-```
-
-The `## 📌 Current Best` block must still be present at the top after archival.
-If it is missing, the `split_entries` function did not handle it correctly —
-restore from the archive and fix the script before continuing.
-
-## Step 5 -- Update global index
-
-Open `LOG.md` and confirm the row for `<task>` still points to `logs/LOG.<task>.md` (the active file -- not the archive).
-
-## Step 6 -- Rebuild the search index
-
-The archive script auto-triggers this, but if it didn't run, do it manually:
-
-```bash
-python scripts/build_search_index.py
-```
-
-This regenerates `logs/archive/INDEX.jsonl` with all entries (active + archived).
-
-## Step 7 -- Refresh STATUS
-
-The hot log just changed; regenerate the compact STATUS file (Claude's default
-read surface):
-
-```bash
-python scripts/generate_status.py --task <task>
-```
-
-## Archive file naming
-
-`logs/archive/LOG.<task>.<YYYY-MM-DD>.<seq>.md`
-
-The `<seq>` suffix (01, 02, ...) prevents collisions if you archive the same task multiple times in one day.
-
-## Pointer line format (written into the active log)
-
-```markdown
-> Archived entries 001--NNN -> logs/archive/LOG.<task>.<date>.<seq>.md
-```
-
-This line is always kept at the **bottom** of the active log, below all `## ` entries.
-Multiple pointer lines accumulate as more archives are created -- do not delete them.
diff --git a/.claude/skills/archive-search/SKILL.md b/.claude/skills/archive-search/SKILL.md
deleted file mode 100644
index 79ac3c828..000000000
--- a/.claude/skills/archive-search/SKILL.md
+++ /dev/null
@@ -1,95 +0,0 @@
----
-name: archive-search
-description: Searches archived log entries across all tasks or a specific task. Use when the user says "what did we try before", "search the archive", "find old runs with X", or when the agent needs historical context beyond the 5 hot entries in the active log.
-allowed-tools: Bash, Read
----
-
-# Search Archived Log Entries
-
-## Step 1 — Identify scope
-
-Determine: is this a single-task search or cross-task?
-
-```bash
-# List all archive files
-ls logs/archive/
-
-# For a specific task:
-ls logs/archive/LOG.<task>.*.md
-
-# Total archive size
-wc -l logs/archive/*.md | tail -1
-```
-
-## Step 1.5 — Check the search index (fast path)
-
-If the search can be answered by metadata alone (run number, date, metric, title keyword, task),
-use the structured index first:
-
-```bash
-python3 -c "
-import json
-for line in open('logs/archive/INDEX.jsonl'):
-    e = json.loads(line)
-    if '<keyword>' in e.get('title','').lower() or '<keyword>' in str(e.get('tags',[])).lower():
-        metric = f\"{e.get('key_metric','')}: {e.get('key_value','')}\" if e.get('key_value') else ''
-        print(f\"{e['date']}  Run {e.get('run','?'):>3}  [{e['task']}]  {e['title']}  {metric}\")
-"
-```
-
-For metric-based queries (e.g. "accuracy > 90%"):
-
-```bash
-python3 -c "
-import json
-for line in open('logs/archive/INDEX.jsonl'):
-    e = json.loads(line)
-    if e.get('key_value') is not None and e['key_value'] > 90:
-        print(f\"{e['date']}  Run {e.get('run','?'):>3}  [{e['task']}]  {e.get('key_metric','')}: {e['key_value']}\")
-"
-```
-
-If this returns sufficient results, you can skip the full-text grep in Steps 2-3.
-
-## Step 2 — Search
-
-```bash
-# Search for a keyword across all archives
-grep -l "<keyword>" logs/archive/*.md
-
-# Search within a specific task's archives
-grep -n "<keyword>" logs/archive/LOG.<task>.*.md
-
-# Find runs matching a config value (e.g. learning rate)
-grep -n "lr=1e-3\|learning_rate: 0.001" logs/archive/LOG.<task>.*.md
-
-# Find runs by approximate date range
-ls logs/archive/LOG.<task>.2025-0[1-3]*.md
-```
-
-## Step 3 — Read matching entries
-
-Once you have a filename and line number, read the surrounding context:
-
-```bash
-# Read a specific archive file
-cat logs/archive/LOG.<task>.<date>.<seq>.md
-
-# Or just the matching section (N lines around match)
-grep -A 20 "<keyword>" logs/archive/LOG.<task>.*.md | head -60
-```
-
-## Step 4 — Summarize findings
-
-Report to the user:
-- Which archive files matched
-- Run numbers and dates of matching entries
-- The key content (config, result, notes) from each match
-- Whether any matching run is a candidate to resume or replicate
-
-## Notes
-
-- Archive files are read-only. Never write to them.
-- If a keyword matches many entries, filter by date or metric value.
-- Active log entries (logs/LOG.<task>.md) are not searched here —
-  the agent can read those directly.
diff --git a/.claude/skills/generate-leaderboard/SKILL.md b/.claude/skills/generate-leaderboard/SKILL.md
deleted file mode 100644
index 782cc0be7..000000000
--- a/.claude/skills/generate-leaderboard/SKILL.md
+++ /dev/null
@@ -1,87 +0,0 @@
----
-name: generate-leaderboard
-description: Reads the Current Best pinned block from every per-task log and produces a ranked leaderboard table across all tasks. Use when the user says "show me the leaderboard", "what is the best result per task", "update the summary table", or "what should I work on next".
-allowed-tools: Read, Write, Bash, Glob
----
-
-# Generate Cross-Task Leaderboard
-
-## Step 1a — Fast path: aggregate run.json sidecars (preferred)
-
-Every promoted run has a flat `run.json`. Aggregate them without touching markdown:
-
-```bash
-python3 -c "
-import json, pathlib
-from collections import defaultdict
-best = defaultdict(lambda: (None, None))  # task -> (value, record)
-for p in pathlib.Path('experiments').rglob('run.json'):
-    try:
-        r = json.loads(p.read_text())
-    except Exception:
-        continue
-    task = r.get('task'); v = r.get('key_value'); km = r.get('key_metric')
-    if task is None or v is None: continue
-    # Lower-is-better for ppl/loss/mae/mse; higher-is-better otherwise
-    lower_better = km in {'test_ppl','val_ppl','mean_ppl','loss','mae','mse'}
-    cur_v, _ = best[task]
-    if cur_v is None or (lower_better and v < cur_v) or (not lower_better and v > cur_v):
-        best[task] = (v, r)
-for task, (v, r) in sorted(best.items()):
-    print(f\"{task:<30} {r.get('key_metric',''):<10} {v:>10}  run={r.get('run_id')}  {r.get('date','')}\")
-"
-```
-
-If a task has no sidecars, fall back to Step 1b.
-
-## Step 1b — Fallback: pinned blocks from STATUS / LOG
-
-```bash
-for f in logs/STATUS.*.md logs/LOG.*.md; do
-  echo "=== $f ==="
-  awk '/^## 📌 Current Best/{found=1} found{print} /^---/{if(found) exit}' "$f"
-  echo ""
-done
-```
-
-Prefer `logs/STATUS.*.md` — smaller and authoritative for the pinned block.
-
-## Step 2 — Build the leaderboard table
-
-For each task, extract the 🥇 row from its pinned block.
-Produce a single markdown table sorted by metric value (best first).
-Group tasks by metric type if they use different metrics.
-
-Output format:
-
-```markdown
-## Leaderboard — <date>
-
-### <metric group> (higher is better / lower is better)
-
-| Rank | Task | Run | Date | Metric | Value | Config |
-|------|------|-----|------|--------|-------|--------|
-| 1 | mnist_cnn | Run 057 | 2025-04-10 | acc | 98.3% | lr=1e-3 |
-| 2 | cifar10 | Run 031 | 2025-03-22 | acc | 94.1% | lr=3e-4 |
-...
-
-### No result yet
-
-Tasks with no quantitative result: <comma-separated list>
-```
-
-## Step 3 — Write output
-
-Save to `experiments/_leaderboard/leaderboard_<YYYY-MM-DD>.md`.
-Also print to stdout for immediate review.
-
-## Step 4 — Embed in PAPER.md (optional)
-
-If the user asks to update PAPER.md, replace the content under
-`## Summary` or `## Current Results` with the leaderboard table.
-Do not touch other sections.
-
-## Step 5 — Bi-directional sync
-
-No LOG entry needed for leaderboard generation unless results changed.
-If you updated PAPER.md, note it in the relevant task's log entry.
diff --git a/.claude/skills/generate-report/SKILL.md b/.claude/skills/generate-report/SKILL.md
deleted file mode 100644
index 239a7aa76..000000000
--- a/.claude/skills/generate-report/SKILL.md
+++ /dev/null
@@ -1,120 +0,0 @@
----
-name: generate-report
-description: Analyzes a promoted experiment directory and produces a structured markdown report under experiments/_reports/. Use after intaking and promoting HPC results to summarize findings. Also use when the user says "generate a report", "summarize results", or "write the experiment report".
-allowed-tools: Read, Write, Bash, Glob, Grep
----
-
-# Generate Experiment Report
-
-## Step 1 -- Identify the experiment and task
-
-Ask (or infer from context): what is the promoted experiment path and which task does it belong to?
-
-```bash
-ls experiments/<task>/
-```
-
-The task name determines which log file to update: `logs/LOG.<task>.md`.
-
-## Step 2 -- Inventory results
-
-```bash
-# Check what's in the experiment root
-ls experiments/<task>/<experiment_name>/
-
-# Find all result/metric files
-find experiments/<task>/<experiment_name>/ -name "*.json" -o -name "*.csv" | head -30
-
-# Check existing reports
-ls experiments/<task>/<experiment_name>/_reports/ 2>/dev/null || echo "No reports yet"
-```
-
-## Step 3 -- Read key files
-
-- `status.json` -- final state and task metadata
-- `manifest.json` -- expected vs observed jobs
-- Per-run metric files (e.g. `results.json`, `metrics.csv`, `summary.json`)
-- Existing `README.md` inside the package
-
-## Step 4 -- Write the report
-
-Output path: `experiments/<task>/<experiment_name>/_reports/<report_name>/<report_name>.md`
-
-### Report structure
-
-```markdown
-# <Experiment Name> -- Report
-
-**Date:** YYYY-MM-DD
-**Task:** <task>
-**Status:** complete / incomplete (N/M runs)
-
-## Summary
-
-One paragraph: what was tested, key result, headline number.
-
-## Configuration
-
-Table of key hyperparameters swept.
-
-## Results
-
-### Main metric table
-
-| Method | Metric | Mean | P10 | Std |
-|--------|--------|------|-----|-----|
-
-### Figures
-
-Embed any generated plots (use relative paths).
-
-### Key findings
-
-Bullet list of the 3-5 most important observations.
-
-## Failure / Incomplete Jobs (if any)
-
-List missing jobs and their impact on conclusions.
-
-## Next Steps
-
-What should be done based on these results?
-```
-
-## Step 5 -- Update canonical summary
-
-Ensure `_reports/<report_name>/summary.json` exists with required fields:
-- `experiment_id`, `source_packages`, `expected_jobs`, `observed_jobs`, `missing_jobs`, `status`
-
-## Step 6 -- Bi-directional sync (required by AGENTS.md)
-
-### PAPER.md
-
-1. In the **Results** section: update the current best result for this task.
-2. If the previous best result is being displaced, move it (with its config) to **Appendix: Ablation History -- \<task\>**.
-3. In **Ablation Summary**: add or update the row for this variant in the per-task table.
-
-```markdown
-<!-- Ablation Summary table format -->
-| Run | Key change | Metric | vs. previous |
-|-----|------------|--------|--------------|
-```
-
-After adding the new row to the Ablation History table, re-sort it:
-
-```bash
-conda activate normal
-python scripts/sort_ablation_tables.py
-```
-
-### README.md
-
-Update the "Current Results" row for this task (one line per task, overwrite -- do not append).
-
-### logs/LOG.\<task\>.md
-
-Invoke `/log-entry` for this task. The entry should reference the report path and headline number.
-
-### LOG.md (global index)
-
-Update the row for `<task>` with the new Last Run, Status, and link.
diff --git a/.claude/skills/hpc-submit/SKILL.md b/.claude/skills/hpc-submit/SKILL.md
deleted file mode 100644
index 53a5b6362..000000000
--- a/.claude/skills/hpc-submit/SKILL.md
+++ /dev/null
@@ -1,256 +0,0 @@
----
-name: hpc-submit
-description: Guides creation of a compliant SLURM submitter + runner script pair for this project. Use when preparing a new experiment for cluster submission, writing new HPC scripts, or auditing existing scripts against project policy. Also use when the user says "create a submitter", "write SLURM scripts", or "submit to cluster".
-allowed-tools: Read, Write, Edit, Bash, Glob, Grep
----
-
-# HPC Submit — New Experiment
-
-Full policy: [docs/policy/HPC_RESULT_INTAKE_POLICY.md](docs/policy/HPC_RESULT_INTAKE_POLICY.md).
-
-**Cluster facts + resilience templates live in the global `pax-hpc` skill**
-(`~/.claude/skills/pax-hpc/`): cluster choice (NEW=login-p02/conda/2d cap;
-OLD=login-prod/Singularity-always/6d cap), VRAM-precise GPU labels
-(`references/new-cluster.md`), and the copy-paste requeue/resume/VRAM-guard
-scaffolding (`references/resilience.md`). This project skill is the *house style*
-on top of that. Verified on both clusters, re-confirmed 2026-07-17: `preempt`
-`GraceTime=0` (no window on a preempt kill — only the last periodic checkpoint
-survives; the 30 s `KillWait` grace is the *walltime*/`scancel` path, not
-preemption), `PreemptMode=REQUEUE` **does not auto-requeue** — so the resilience
-contract below is mandatory, not optional. **Preempt jobs also require
-`--qos=preempt`** (not just `-p preempt`): without it the job silently runs under
-the `normal` QOS (cpu=250) and gets none of the higher preempt ceiling. The
-`normal` and `preempt` GPU caps are separate 10-GPU pools, so preempt spillover
-buys up to 20 concurrent GPUs (verified 2026-07-17).
-
-## Script pair overview
-
-Every experiment needs exactly two scripts:
-
-| Script | Runs on | Does |
-|---|---|---|
-| `scripts/<task>/submit_<task>_<variant>.sh` | Gateway/login node | `sbatch` orchestration only |
-| `scripts/<task>/run_<task>_<variant>.sh` (or `.py`) | Compute node via SLURM | Training, logging, packaging |
-
-## Submitter checklist (`bash` on gateway)
-
-- [ ] `set -euo pipefail` at top
-- [ ] Verifies `sbatch` is available
-- [ ] Validates required files/paths before submission
-- [ ] Prints usage and key env overrides
-- [ ] GPU constraint default uses **VRAM-precise labels**, not bare type names:
-      `a100-80G|h100-80G|h200-141G|l40s-48G` (bare `a100` silently admits the
-      40 GB card → OOM). Use the `constraint_for_tier` recipe from the global
-      `pax-hpc` skill's `references/new-cluster.md`.
-- [ ] Supports override via `GPU_CONSTRAINT` or `SBATCH_CONSTRAINT` env vars
-- [ ] Constraint fallback: try full VRAM-precise alternation → retry token-by-token → retry without `--constraint`
-- [ ] **Resilience SBATCH directives on EVERY job** (walltime trap is universal —
-      see contract below): `--requeue`, `--signal=B:USR1@30`, `--open-mode=append`
-- [ ] **`preempt` jobs pass BOTH `--partition=preempt` and `--qos=preempt`** — the
-      QOS flag is required or you silently fall back to the `normal` ceiling (cpu=250)
-- [ ] `--time` within the cap of the target cluster (**2d new / 6d old**); never rely on the 15-min default
-- [ ] **No hardcoded `--exclude`** — bad nodes are learned at runtime by the runner
-- [ ] Token rotation: when splitting across tokens, rotate preference across jobs (not all on first accepted token)
-- [ ] One SLURM job per independent sweep point (mode × seed grid) — no bundling unless true dependency
-- [ ] Big / multi-stage work uses `--dependency` chains, not one mega-job
-- [ ] Does NOT run Python, training, plotting, or `singularity exec` on gateway
-
-## Runner checklist (compute node)
-
-- [ ] Writes only inside one unique package root per task (no out-of-package writes)
-- [ ] Emits all four required files inside package root:
-  - `status.json` with `task_id` and `state` (`finished`/`partial`/`failed`)
-  - `manifest.json` with `required_files` and `promote_to`
-  - `README.md` (human-readable run context)
-  - `INTAKE.md` (local post-transfer commands)
-- [ ] Uses fail-fast validation for required runtime inputs
-- [ ] Verifies required Python modules before training
-- [ ] Records bootstrap/failure outcome in package metadata
-- [ ] Logs node/hardware info at startup (see **Hardware & timing log block** below)
-- [ ] Logs job start time and wall-clock end time (success and failure paths)
-- [ ] Registers EXIT trap that logs reason + elapsed time on any exit (OOM, kill, error)
-- [ ] **VRAM guard**: read `nvidia-smi` at startup; if VRAM `< MIN_GPU_MEM_MB`,
-      append the node to `ExcNodeList` and `scontrol requeue` (don't just requeue —
-      exclude, or it loops onto the same small card)
-- [ ] **CUDA-init guard**: `torch.zeros(1, device="cuda")` probe AFTER env activate;
-      on failure, exclude-node + requeue (a healthy `nvidia-smi` can still have a dead CUDA ctx)
-- [ ] **Signal traps** (`SIGUSR1` + `SIGTERM`) on EVERY job: requeue on the walltime
-      warning (universal — a job that overruns `--time` must requeue, never die) AND on
-      preemption; on preemption requeue *first* (GraceTime=0 → no window). Gate the
-      requeue loop on a `DONE` sentinel so it stops once the work is complete. See contract below.
-- [ ] **Robust resume**: resolve the resume dir by probing for the **highest-step
-      checkpoint** across `run_*/`, not a single sentinel file; honor `RESUME_RUN_DIR`
-- [ ] **Deliverable checkpoints off by default** (see **Checkpoint policy** below):
-  - Training code does not call `torch.save` / `save_pretrained` / Keras `save_best=True`
-    for *deliverable weights* unless gated by `--save-checkpoint` (or `KEEP_CHECKPOINT=1`), default **off**
-  - `required_files` in manifest.json omits any `*.pt`/`*.pth`/`*.safetensors`/`*.bin` entries
-  - **EXCEPTION — resume checkpoints are mandatory for preempt/long jobs**: a single
-    latest, atomically-written (`*.tmp` → `mv`) checkpoint every ≤~30 min of compute,
-    **deleted on clean finish**. This is transient resume state, not a deliverable, so
-    it stays out of `required_files` but is required for the job to survive `GraceTime=0`.
-
-## Walltime policy (user directive 2026-07-11 — HARD RULE)
-
-- **`--time=23:55:00` is the MAXIMUM walltime for any job.** Do not request more.
-- **Every task MUST be able to requeue itself** on timeout OR on an HPC kill, and
-  resume from its last checkpoint — never from step 0. A job that cannot save and
-  resume cannot be submitted.
-- Rationale (measured): a tied-LoRA run at ~270M on FineWeb-Edu took **22h31m**;
-  a 14h walltime would have failed it. Long jobs are normal, so the 23:55 cap +
-  self-requeue is what makes them finish. Jobs longer than 23:55 complete by
-  requeueing across multiple windows, resuming from the periodic checkpoint.
-- This means the periodic atomic checkpoint (≤ ~30 min) is **mandatory for every
-  job**, not just preempt/near-cap ones — it is what a requeue resumes from.
-
-## Resilience contract (walltime trap MANDATORY for ALL jobs)
-
-> **User directive (2026-06-10): every HPC task traps the walltime SIGTERM and
-> auto-requeues until completed — no exceptions.** Walltime estimates are
-> routinely wrong; an overrun must resume, never silently die. The periodic
-> *checkpoint* layer below stays scoped to preempt / near-cap jobs (it's what
-> lets a requeue resume near where it died); the *trap + `--requeue`* layer is
-> universal. The requeue loop is gated on a `DONE` sentinel — requeue while
-> unfinished, stop once the run completes.
-
-
-Copy-paste bash/Python templates: global `pax-hpc` skill →
-`references/resilience.md`. The canonical in-repo example is
-[scripts/wikitext/run_wikitext_v6_slurm.sh](scripts/wikitext/run_wikitext_v6_slurm.sh)
-(already implements `--requeue`, USR1/TERM handlers, VRAM guard, cuda-exclude).
-
-Why mandatory: `preempt` has `GraceTime=0` on both clusters — a preempted job is
-SIGKILLed with **no checkpoint window**, and `PreemptMode=REQUEUE` does **not**
-auto-requeue unless the job is `--requeue`-eligible. Several tasks also exceed the
-cap (AWD-LSTM ≈3d vs the new 2d cap), so they *cannot finish* without resume.
-
-The contract has three layers — get all three or the job is not resilient:
-
-1. **Eligibility (submitter):** `#SBATCH --requeue` + `--signal=B:USR1@30` +
-   `--open-mode=append`, `--time` within the cluster cap. On `preempt`, also
-   `#SBATCH --qos=preempt` (required for the higher ceiling; a bare `-p preempt`
-   falls back to the `normal` QOS).
-2. **Survival (runner, bash):** trap `SIGUSR1`/`SIGTERM` → on preemption requeue
-   *first* then best-effort checkpoint, on walltime checkpoint-then-requeue;
-   VRAM-guard + cuda-guard that exclude-node + requeue; resume by highest-step probe.
-3. **State (trainer, Python):** trap `SIGUSR1` → write latest checkpoint → exit;
-   **periodic atomic checkpoint every ≤~30 min** (the only thing that survives a
-   0-grace preemption); resume from the highest-step checkpoint; delete on clean finish.
-
-Requeue lives in the **bash** layer (survives a hung trainer); the Python layer only
-checkpoints. The walltime trap + `--requeue` are NOT optional for any job —
-CPU-only and sub-hour jobs on `batch`/`gpu` still carry them (a wrong runtime
-estimate must requeue, not die); they may skip only the *periodic checkpoint*
-layer if a from-scratch restart on requeue is acceptable.
-
-## Checkpoint policy
-
-> Full policy: [docs/policy/HPC_RESULT_INTAKE_POLICY.md](docs/policy/HPC_RESULT_INTAKE_POLICY.md)
-> section **Checkpoint Persistence Policy**.
-
-**Default**: evaluation-only experiments persist config + metrics + history,
-not weights. `config + code commit + seed = same result`; checkpoints add no
-reproducibility and a lot of disk.
-
-**Opt in** (explicit flag, default off) only when:
-1. The weight artifact itself is the deliverable (released model, teacher, warmstart)
-2. Multi-stage training needs mid-run state (ReLoRA merge, warmstart handoff)
-3. Active investigation needs post-hoc weight probing (eigenvalues, activations)
-
-When opting in, record `extra.checkpoint_reason` in manifest.json and keep the
-file optional (never in `required_files`) so evaluation reruns don't trip
-`INCOMPLETE`.
-
-**Cleanup of legacy dirs**: `python scripts/strip_checkpoints.py --scan experiments/<task>/`
-removes `*.pt`/`*.pth`/`*.safetensors`/`*.bin` while preserving all metadata.
-
-## Hardware & timing log block
-
-Paste this block **once, near the top of every runner script**, immediately after the SLURM env debug section:
-
-```bash
-# =============================================================================
-# HARDWARE & TIMING INFO
-# =============================================================================
-JOB_START_TIME=$(date +"%Y-%m-%dT%H:%M:%S")
-JOB_START_EPOCH=${SECONDS}
-echo "===== JOB START: ${JOB_START_TIME} ====="
-echo "Node:       ${SLURMD_NODENAME:-unknown}"
-echo "NodeList:   ${SLURM_JOB_NODELIST:-unknown}"
-echo "JobID:      ${SLURM_JOB_ID:-unknown}"
-echo "ArrayTask:  ${SLURM_ARRAY_TASK_ID:-none}"
-echo "Cluster:    $(hostname -d 2>/dev/null || hostname)"
-echo "SLURM job: ${SLURM_JOB_ID}"
-
-# GPU info (model + VRAM)
-if command -v nvidia-smi &>/dev/null; then
-    echo "--- GPU ---"
-    nvidia-smi --query-gpu=index,name,memory.total,driver_version \
-        --format=csv,noheader,nounits 2>/dev/null \
-        | awk -F',' '{printf "  GPU %s: %s  |  VRAM: %s MiB  |  Driver: %s\n",$1,$2,$3,$4}'
-else
-    echo "GPU: not available"
-fi
-
-# CPU memory
-echo "--- CPU Memory ---"
-free -h | grep -E '^Mem'
-
-# Register EXIT trap: always log finish time + status + elapsed
-_log_job_exit() {
-    local exit_code=$?
-    local elapsed=$(( SECONDS - JOB_START_EPOCH ))
-    local end_time
-    end_time=$(date +"%Y-%m-%dT%H:%M:%S")
-    echo ""
-    echo "===== JOB EXIT: ${end_time} ====="
-    echo "Exit code:    ${exit_code}"
-    echo "Elapsed:      ${elapsed}s  (~$(( elapsed/3600 ))h $(( (elapsed%3600)/60 ))m $(( elapsed%60 ))s)"
-    if [ "${exit_code}" -eq 0 ]; then
-        echo "Status:       SUCCESS"
-    elif [ "${exit_code}" -eq 137 ]; then
-        echo "Status:       KILLED (OOM or external kill signal — check memory limits)"
-    else
-        echo "Status:       FAILED (see Python traceback above)"
-    fi
-    echo "Start:        ${JOB_START_TIME}"
-    echo "End:          ${end_time}"
-}
-trap '_log_job_exit' EXIT
-# =============================================================================
-```
-
-**Debug notes captured by this block:**
-- GPU model + VRAM → tells you if a different GPU type ran (A100 vs L40s affects numerics/OOM thresholds)
-- Exit code 137 → OOM kill (not a Python crash); increase `--mem` or reduce batch size
-- Elapsed time → compare against `--time` budget; if close, job was likely preempted or killed at limit
-- Python loss/parameter logs → emitted to the SLURM `.out` file by the training script; grep `loss` or `param` in that file to trace divergence
-
-## Policy alignment checklist
-
-- [ ] Config/wrapper paths remain under task-scoped roots (not repo root)
-- [ ] Script behavior aligns with `scripts/validate_experiment_policy.py`
-- [ ] Docs synchronized: `README.md`, `PAPER.md`, `LOG.md`
-- [ ] Aggregation of split-job outputs happens locally after intake
-
-## Common failure patterns to avoid
-
-1. `Invalid feature specification` — use bracket constraint with token fallback
-2. Running `bash submitter.sh` instead of `sbatch submitter.sh` (or vice versa)
-3. Python/container execution on gateway node
-4. Shared HF cache across parallel jobs — use per-job `HF_HOME` or `TRANSFORMERS_CACHE`
-5. Non-self-contained outputs (writes outside package root)
-6. **Silent OOM from a bare `a100` constraint** landing on a 40 GB card — use a
-   VRAM-precise label (`a100-80G`) + the runtime VRAM guard with `MIN_GPU_MEM_MB`
-7. **Preempted job vanishes instead of requeuing** — missing `#SBATCH --requeue`
-   (`PreemptMode=REQUEUE` alone does not requeue your job)
-8. **Resume starts from zero after ≥2 requeues** — trusting a stale sentinel file
-   instead of probing for the highest-step checkpoint
-
-## After writing scripts
-
-Run the policy validator:
-```bash
-python scripts/validate_experiment_policy.py
-```
-
-Then invoke `/log-entry` to record the new scripts.
diff --git a/.claude/skills/intake-results/SKILL.md b/.claude/skills/intake-results/SKILL.md
deleted file mode 100644
index 807ad891e..000000000
--- a/.claude/skills/intake-results/SKILL.md
+++ /dev/null
@@ -1,101 +0,0 @@
----
-name: intake-results
-description: Runs the full HPC result intake pipeline for this project. Use when experiment results have been transferred from the cluster into experiments/from_HPC and need to be validated, promoted, and archived. Also use when the user says "intake", "promote results", or "process HPC output".
-allowed-tools: Bash, Read, Write, Edit, Glob, Grep
----
-
-# Intake Results from HPC
-
-Full policy is in [docs/policy/HPC_RESULT_INTAKE_POLICY.md](docs/policy/HPC_RESULT_INTAKE_POLICY.md).
-
-## Step 1 -- Check what arrived
-
-```bash
-ls experiments/from_HPC/
-```
-
-If packages landed under a staging subdirectory (`hpc_staging`), normalize first:
-
-```bash
-bash scripts/hpc/intake_from_hpc_staging.sh --dry-run
-# If dry-run looks correct:
-bash scripts/hpc/intake_from_hpc_staging.sh --execute --run-intake --interactive
-```
-
-```bash
-# Extract SLURM job IDs from stdout logs in the incoming package
-grep -r "SLURM job:" experiments/from_HPC/ 2>/dev/null | head -20
-```
-
-Copy the job ID(s) into the log entry SLURM field when writing the entry.
-
-## Step 2 -- Run intake (dry-run first, always)
-
-```bash
-python scripts/hpc/hpc_result_intake.py --interactive --dry-run
-```
-
-Review the output, then run live:
-
-```bash
-python scripts/hpc/hpc_result_intake.py --interactive
-```
-
-## Step 3 -- Decide on each package
-
-| Classification | Condition | Action |
-|----------------|-----------|--------|
-| `COMPLETE` | `status.json` valid, `state=finished`, all required files present | Auto-promote |
-| `INCOMPLETE` | `state=partial/failed` or required files missing | Choose: rerun missing, rerun all, accept partial, or skip |
-| `UNKNOWN` | `status.json` missing or malformed | Do NOT silently promote -- investigate first |
-
-Quarantine path (if promotion fails): `experiments/_quarantine/<name>_<timestamp>/`
-Audit log: `experiments/from_HPC/intake_events.jsonl`
-
-## Step 4 -- Unify results
-
-```bash
-python scripts/hpc/unify_hpc_results.py --dry-run
-python scripts/hpc/unify_hpc_results.py --archive-complete
-```
-
-## Step 5 -- Identify the task
-
-From the promoted package, determine the task name. This is the subdirectory under `experiments/` where the package landed. All subsequent log writes target `logs/LOG.<task>.md`.
-
-## Step 6 -- Generate report
-
-After promotion, invoke `/generate-report` with the promoted experiment path. That skill handles PAPER.md, README.md, and the per-task log in one step.
-
-## Step 7 -- Post-intake refresh (single command)
-
-Run the wrapper. It writes `run.json` sidecars for the task's runs, archives
-`LOG.<task>.md` if it exceeds 10 active entries, regenerates STATUS, rebuilds
-`LOG.md`, and rebuilds the archive search index.
-
-```bash
-python scripts/post_intake_refresh.py --task <task>
-```
-
-For a multi-task intake pass the flag more than once, or omit `--task` to
-refresh every active task:
-
-```bash
-python scripts/post_intake_refresh.py --task wikitext --task surrogate
-python scripts/post_intake_refresh.py       # all tasks
-```
-
-This replaces the previous manual sequence of `write_run_sidecar.py` +
-`archive_log.py` + `generate_status.py` + `refresh_index.py` +
-`build_search_index.py`.
-
-## Step 8 -- Bi-directional sync (required by AGENTS.md)
-
-The `/generate-report` skill covers this. Confirm before closing the session:
-- [ ] `logs/LOG.<task>.md` has a new entry
-- [ ] `logs/STATUS.<task>.md` regenerated
-- [ ] `run.json` sidecars present for every promoted run
-- [ ] `LOG.md` global index row is updated
-- [ ] PAPER.md Results section is current
-- [ ] PAPER.md Ablation Summary row exists for this variant
-- [ ] README.md "Current Results" row is current
diff --git a/.claude/skills/log-entry/SKILL.md b/.claude/skills/log-entry/SKILL.md
deleted file mode 100644
index 7a7b8ef79..000000000
--- a/.claude/skills/log-entry/SKILL.md
+++ /dev/null
@@ -1,205 +0,0 @@
----
-name: log-entry
-description: Appends a new structured entry to LOG.md following the project's experimental log format. Use after completing an experiment run, implementing a feature, fixing a bug, or any change that should be recorded. Also use when the user says "update the log", "add a log entry", or "record this run".
-allowed-tools: Read, Edit
----
-
-# Add a Log Entry
-
-## Step 0 — Check and update the pinned block
-
-After writing the entry (Step 3), decide whether this run is a new best or
-second-best for this task:
-
-- Read the existing `## 📌 Current Best` block (if any)
-- Compare the new result against the current 🥇 and 🥈
-- If the new result is better than 🥇: demote current 🥇 to 🥈, set new 🥇
-- If the new result is better than 🥈 only: replace 🥈
-- If the new result is not top-2: leave the block unchanged
-- If no block exists yet: create it with this run as 🥇 (and 🥈 empty)
-
-**The pinned block must always appear before the first `## ` entry in the file.**
-When in doubt about ranking (e.g. the metric changed), leave the block as-is
-and note "metric changed — manual review needed" in the Notes row.
-
-### PAPER.md sync (required when pinned block changes)
-
-If the pinned block was updated (🥇 changed):
-
-1. Open `PAPER.md` and locate the Results section for this task.
-   It will be under `## Results` or a subsection like `### <task>`.
-2. Replace the current best result line with the new 🥇 values.
-3. Move the displaced result (the old 🥇) into the task's
-   `## Appendix: Ablation History — <task>` table as a new row.
-   Format: `| YYYY-MM-DD | Run NNN | <what changed> | <metric> | <value> |`
-4. Do not rewrite the whole Results section — surgical edit only.
-
-If the metric or task does not appear in PAPER.md Results yet, add it.
-If PAPER.md does not exist, skip this step and note it in the log entry.
-
-## Step 1 -- Identify the task
-
-Determine the task name from context (e.g. the experiment directory, user instruction,
-or the most recently modified code). The task name is the subdirectory under `experiments/`.
-
-The target file is `logs/LOG.<task>.md`.
-
-## Step 2 -- Read the current log
-
-```bash
-head -10 logs/LOG.<task>.md
-```
-
-Find the current run number from the first `## ` heading. New run number = last + 1.
-
-## Step 3 -- Write the entry
-
-Use today's date (YYYY-MM-DD). **Prepend** the new entry immediately after the file
-header and before the first existing `## ` heading -- logs are reverse-chronological
-(newest first).
-
-### Entry format
-
-```markdown
-## YYYY-MM-DD Run NNN (Short Title -- Key Distinction)
-
-```yaml meta
-run: NNN
-date: "YYYY-MM-DD"
-task: <task_name>
-type: experiment          # experiment | infrastructure | bugfix | intake
-slurm_job: <job_id>       # omit if not a SLURM run
-key_metric: <metric_name> # omit if no quantitative result
-key_value: <number>       # the single most important result
-key_unit: "%"             # omit if dimensionless
-walltime_mean_s: <number> # mean wallclock seconds across all tasks; omit if not a SLURM run
-walltime_std_s: <number>  # std of wallclock seconds across all tasks; omit if not a SLURM run
-params:
-  model: <model_name>
-  rank: <rank>
-  method: <method>
-  epochs: <N>
-  seeds: <N>
-tags: [<tag1>, <tag2>]
-```
-
-**SLURM job:** `<job_id>` | **Array:** `<array_range>` | **Cluster:** `<cluster_name>`
-
-### Objective
-
-One paragraph explaining what this run/change set out to do and why.
-
-### Differential Update
-
-**`path/to/changed/file.py`** (brief scope summary):
-- Bullet explaining each specific change and rationale
-
-**`configs/task/config.yaml`**:
-- What changed and why
-
-### Wallclock (required for SLURM runs)
-
-For any entry that corresponds to one or more SLURM jobs, always include a
-`### Wallclock` section in the entry body. Extract elapsed seconds from the
-`.out` files (the `Elapsed: <N>s` line written by the runner) and compute
-mean and std across all tasks in the sweep:
-
-```bash
-grep -h "Elapsed:" experiments/from_HPC/<task>/<prefix>.*.out \
-  | awk '{print $2}' | sed 's/s//' \
-  | python3 -c "
-import sys, statistics
-vals=[int(l) for l in sys.stdin]
-print(f'N={len(vals)}  mean={statistics.mean(vals):.0f}s  std={statistics.stdev(vals):.0f}s  min={min(vals)}s  max={max(vals)}s')
-"
-```
-
-Format the section as:
-
-```markdown
-### Wallclock
-
-**Overall (N=<N> tasks):** mean=<mean>s (<mean_h>h) ± <std>s (<std_h>h) | min=<min>s | max=<max>s
-
-| Config | N | mean (s) | std (s) | mean (h) |
-|---|---|---:|---:|---:|
-| <config> | <n> | <mean> | <std> | <h>h |
-```
-
-Include per-config breakdown whenever the sweep spans multiple configurations.
-Omit if all tasks share the same config (report overall only).
-
-### Smoke Test (if applicable)
-
-```
-result_1: shape OK
-result_2: value OK
-```
-
-Brief interpretation.
-
----
-```
-
-### Metadata block rules
-
-The ` ```yaml meta ``` ` block is **required** for every entry:
-- Always include `run`, `date`, `task`, `type`
-- Include `key_metric`/`key_value`/`key_unit` when the entry reports a quantitative result
-- Include `slurm_job` when applicable
-- Include relevant hyperparameters in `params` (free-form dict, task-specific)
-- Add descriptive `tags` for filtering (e.g. `rank-sweep`, `promoted`, `negative-result`)
-
-The `**SLURM job:**` line is optional — omit it for non-SLURM runs (local tests, debug runs).
-When present it must be the first line of the entry body, before Objective.
-
-## Step 4 -- Check if archiving is needed
-
-```bash
-count=$(grep -c "^## " logs/LOG.<task>.md)
-echo "$count entries"
-```
-
-If count > 10, invoke `/archive-log` with `--task <task> --keep-last 5`.
-(Hot logs stay small so STATUS + grep remain fast.)
-
-## Step 5 -- Update the global index
-
-Open `LOG.md` and update the row for this task with the new Last Run and Status.
-
-## Step 6 -- Bi-directional sync (required)
-
-After writing the log entry, verify that PAPER.md, README.md, and code are
-consistent with what was logged. Any methodological change in code must be
-reflected in PAPER.md in the same session.
-
-## Step 7 — Never archive the pinned block
-
-The `## 📌 Current Best` block is permanent. Confirm it is still present at
-the top of the file after any archival operation:
-
-```bash
-head -5 logs/LOG.<task>.md | grep "📌"
-```
-
-If it is missing, restore it from the archive file's header or from memory.
-
-## Step 8 — Refresh STATUS, global index, and search index
-
-After every log entry, regenerate the STATUS file (what Claude loads by default),
-the global index, and the search index from ground truth:
-
-```bash
-conda activate normal
-python scripts/generate_status.py --task <task>
-python scripts/refresh_index.py
-python scripts/build_search_index.py
-```
-
-This replaces the manual row-update instruction. Do not hand-edit LOG.md or
-STATUS.<task>.md — both are regenerated from the per-task log.
-
-## Image embedding
-
-If embedding images, copy to `log/img/[original_name]_[YYYY-MM-DD-H_M_S].[ext]`
-first, then embed with a relative path.
diff --git a/.claude/skills/pax-hpc/SKILL.md b/.claude/skills/pax-hpc/SKILL.md
deleted file mode 100644
index a3a95d06e..000000000
--- a/.claude/skills/pax-hpc/SKILL.md
+++ /dev/null
@@ -1,292 +0,0 @@
----
-name: pax-hpc
-description: >-
-  Prepare and launch compute work on the user's two Tufts "Pax" SLURM HPC
-  clusters. Use this skill WHENEVER the user wants to run, batch, submit,
-  array, schedule, profile, or GPU-accelerate any job, experiment, script, or
-  training run that is meant for the cluster — even if they don't say the word
-  "cluster". Trigger on mentions of sbatch, srun, salloc, squeue, slurm,
-  "the cluster", "on pax", "submit a job", "run this on a GPU", singularity /
-  apptainer containers, conda environments meant for the cluster, login-p02,
-  login-prod-03, preempt / requeue / resume, or any request to turn local code
-  into something that runs on HPC. There are two clusters: OLD (login-prod-03,
-  RHEL7, runs programs via Singularity) and NEW (login-p02, Rocky9, runs
-  programs via conda). Always ask which one first.
----
-
-# Pax HPC (two clusters)
-
-This skill captures how the user works with their two Tufts Pax SLURM clusters
-so they never have to re-explain it. Both clusters share ONE filesystem, so
-data, repos, conda envs, and Singularity images live in the same place on both.
-The clusters differ in OS, software runtime, walltime limits, and GPU hardware.
-
-| | OLD cluster | NEW cluster |
-|---|---|---|
-| Login host | `login-prod-0X.pax.tufts.edu` (01, 03, …) | `login-p02.pax.tufts.edu` |
-| Compute nodes | legacy `cc1gpu / s1cmp / p1cmp / d1cmp###` | `pax001`–`pax120` |
-| OS / Slurm | RHEL 7.5 / Slurm 23.02 | Rocky 9.6 / Slurm 23.11 |
-| Runs programs via | **Singularity** (`module load singularity`) | **conda** (`conda activate`) |
-| Max walltime | 6 days | **2 days** |
-| GPUs | p100, v100, t4, a100(40G), + some newer | h100, h200, a100(40/80G), l40s, + more |
-| Why not the other runtime | host glibc too old for modern conda binaries | conda is native; apptainer also available |
-
-Username: `hhazan01`. Full hardware, partitions, QOS, and GPU **VRAM-tagged
-feature labels** are in `references/old-cluster.md` and
-`references/new-cluster.md` — read the relevant one when sizing a job.
-Resilience mechanics (requeue, resume, preempt signal-trap, walltime
-self-requeue, adaptive node-exclude, VRAM guard, dependency chains) live in
-`references/resilience.md`. Bringing results back and classifying them lives in
-`references/intake.md`.
-
-## The non-negotiable rules
-
-1. **Never run ANY task on a login node — under any circumstances** (OLD or NEW).
-   This is absolute (user directive, 2026-06-07). No training, no data crunching,
-   no `python long_thing.py`, no compiling, no `singularity exec` of a workload —
-   **and also no "lightweight" inline work**: no `python scripts/intake.py`,
-   `consolidate.py`, results-aggregation, jsonl parsing, or env-python one-liners
-   on the login node, even for a quick verification. If you need to run code
-   against cluster data, submit it (`sbatch`) or use an interactive compute
-   allocation (`srun --pty`) — or pull the data local and run it here. The login
-   node is ONLY for: editing files, submitting jobs (`sbatch`), launching
-   allocations (`srun --pty`/`salloc`), monitoring (`squeue`, `sacct`,
-   `scancel`), small `ls`/`tail`/`grep` on output files, and `rsync`. **This
-   applies to `login-p02` even when it is used as a no-Duo ProxyJump gateway to
-   the OLD cluster — gateway convenience is not a compute license.** When in
-   doubt, it does not run on login: pull the data local and run it here, or
-   `sbatch`/`srun`.
-
-2. **Always ask which cluster first: OLD or NEW?** The answer changes the
-   runtime wrapper (Singularity vs conda), the walltime ceiling (6d vs 2d), and
-   which GPUs are valid. Do not guess. (If the user already said this session,
-   don't re-ask.)
-
-3. **Prepare everything locally.** Build the job script and any code/config in a
-   local staging folder. Do not assume you can write on the cluster.
-
-4. **The user does the rsync** unless they explicitly say otherwise. Give them
-   the exact `rsync` command, but they run it.
-
-5. **End every preparation with ONE line** the user pastes on the login node to
-   launch — normally `sbatch <script>` (or an array/dependency line). That single
-   line is the deliverable of local prep.
-
-6. **The preempt contract — EVERY `preempt` job carries it, no exceptions**
-   (user directive, 2026-06-20). A job on the `preempt` partition that is not
-   wired for kill→requeue→resume is a bug, not a job. The full contract:
-   - **`#SBATCH --requeue`** — makes the job requeue-eligible. Without it,
-     preemption *cancels* the job (it does not come back). Mandatory.
-   - **Trap the kill signals SLURM sends to stop the process** — the runner
-     traps **`SIGUSR1` + `SIGTERM` + `SIGCONT`** (request the advance warning
-     with `#SBATCH --signal=B:USR1@30`) and routes them to one handler that
-     `scontrol requeue "$SLURM_JOB_ID"`s **first**, then forwards a checkpoint
-     request to the trainer. Requeue first so a slow checkpoint can never cost
-     you the restart. Do the `scontrol requeue` in bash, the checkpoint in
-     Python (§0.1 of resilience.md).
-   - **Auto-requeue until done** — the requeued job restarts the batch script
-     from the top and keeps going until a completion (`DONE`) sentinel exists;
-     `SLURM_RESTART_COUNT` caps runaway loops.
-   - **Periodic checkpoint + resume-from-last is MANDATORY, not optional.**
-     `GraceTime=0` on `preempt` (both clusters, verified) means preemption gives
-     **no on-signal checkpoint window at all** — SIGTERM and SIGKILL arrive
-     back-to-back. The *only* thing that survives is the last **periodic
-     checkpoint** (write every K steps, ≤~30 min of compute at risk; write
-     atomically — tmp + `os.replace`). On restart the job MUST detect the
-     checkpoint and **resume from the last completed step**, never from step 0.
-     A preempt job whose code cannot save and resume from a checkpoint cannot
-     run on `preempt` — fix the resume path first or use a non-preempt partition.
-   - Verified facts to size against: `PreemptMode=REQUEUE`, `GraceTime=0`,
-     `KillWait=30`. Full copy-paste scaffolding in `references/resilience.md`
-     §0–§2.
-
-7. **EVERY task auto-requeues on walltime until it finishes — but ONLY on
-   walltime, and NEVER a crash, a cancel, or a completed job** (user directive,
-   2026-06-10/-12). Walltime estimates are routinely wrong; a job that overruns its
-   `--time` must resume, not die. Every runner carries `#SBATCH --requeue` +
-   `--signal=B:USR1@N`, **traps `SIGUSR1` + `SIGTERM` + `SIGCONT`**, forwards a
-   checkpoint request to the child, and after the child is reaped requeues
-   **iff: (a) the `SIGUSR1` walltime warning fired (`WALLTIME=1`) AND (b) no
-   completion sentinel (unfinished)**. The requeue is **rc-independent**: a walltime
-   stop may exit 0 (checkpointed in time) or non-zero (hard-killed) — `WALLTIME=1`
-   already excludes crashes and `scancel` (neither sends `SIGUSR1`), and the sentinel
-   excludes a clean finish. Trapping `TERM`/`CONT` only keeps the shell alive to make
-   that decision; a `scancel` (also `TERM`) carries `WALLTIME=0`, so it is never
-   requeued. `SLURM_RESTART_COUNT` caps the loop. Preempt-partition jobs differ (no
-   warning) — see §1b. Refs: `references/resilience.md` §0.2, §0.3, §1, §1b.
-
-## Standard workflow
-
-1. **Ask: OLD or NEW cluster?** Then read that cluster's reference file.
-2. **Clarify the job shape**: CPU or GPU, cores, memory, expected walltime,
-   single job vs array vs dependency chain, which conda env (new) or `.sif`
-   image (old). Size it against the reference file's limits.
-3. **Pick the GPU robustly** (see *GPU requests* below) — by VRAM constraint,
-   not by bare typed gres.
-4. **Wire resilience — always.** Every job gets the walltime SIGTERM trap +
-   `--requeue` + resume scaffold from `references/resilience.md` (rule 7).
-   `preempt` jobs and jobs near the cap additionally need the periodic
-   checkpoint (≤~30 min at risk) so they resume near where they died, not from
-   step 0.
-5. **Write the script(s) locally** in a staging dir (default `./<project>_job/`).
-6. **Give the rsync command** to the shared project space. Default destination:
-   `/cluster/tufts/levinlab/hhazan01/<project>/`.
-7. **Give the one launch line**, e.g.
-   `cd /cluster/tufts/levinlab/hhazan01/<project> && sbatch run.sbatch`.
-8. **Tell them how to pull results back** (`references/intake.md`).
-
-## Paths (shared across BOTH clusters)
-
-- **Home:** `/cluster/home/hhazan01` — **small (~28 GB quota, usually near
-  full)**. Do NOT stage data, datasets, package caches, or run output here. Use
-  it only for dotfiles and small scripts.
-- **Lab / project space:** `/cluster/tufts/levinlab/hhazan01` — main working
-  area. Conda envs, Singularity images, git repos, datasets, and staged jobs all
-  live here. `~/cluster` already symlinks to it.
-- **Scratch:** `/cluster/scratch` — large, fast, for transient run output.
-- **Conda envs (new):** `/cluster/tufts/levinlab/hhazan01/miniconda3/envs/*`
-  (verified 2026-05-31: `betse`, `bindsnet`, `delayW`, `dt-d4rl`, `LoRa`).
-- **Singularity images (old):** `/cluster/tufts/levinlab/hhazan01/singularity/*.sif`
-- `umask 0007` is already in `.bashrc` (group-shared lab files).
-
-Because home is tiny, always point conda package dirs, HuggingFace caches, torch
-extensions, and dataset downloads into the lab/scratch space, not home. Do not
-write large checkpoints unless the weight artifact is itself the deliverable —
-config + code commit + seed reproduces a result; checkpoints just burn disk.
-(Checkpoints written *for resume* are exempt — keep only the latest, delete on
-clean finish.)
-
-## Job script templates
-
-Both clusters use the same SBATCH header style; only the run wrapper differs.
-Keep `--time` within the ceiling (6d old, **2d new**).
-
-### Shared SBATCH header
-
-```bash
-#!/usr/bin/env bash
-#SBATCH --job-name=NAME
-#SBATCH --output=./res_%j_%N.txt
-#SBATCH --error=./res_%j_%N.err
-#SBATCH --time=00-04:00:00          # DD-HH:MM:SS  (<=2d new, <=6d old)
-#SBATCH --partition=PARTITION       # see reference file
-#SBATCH --nodes=1
-#SBATCH --ntasks=1
-#SBATCH --cpus-per-task=4
-#SBATCH --mem=16G
-## GPU jobs: see "GPU requests" below — prefer --gres=gpu:1 + --constraint=<VRAM label>
-## Arrays add, e.g.:    #SBATCH --array=1-20   (MaxArraySize=2000)
-## Resilient jobs add:  #SBATCH --signal=B:USR1@30  --open-mode=append  --requeue
-```
-
-### NEW cluster (login-p02) — conda
-
-```bash
-source ~/.bashrc
-conda activate ENV_NAME          # e.g. LoRa, delayW, betse, bindsnet, dt-d4rl
-cd /cluster/tufts/levinlab/hhazan01/PROJECT
-python my_script.py --args ...
-```
-
-### OLD cluster (login-prod-03) — Singularity
-
-Conda will NOT run here (host glibc too old). Wrap the workload in the container:
-
-```bash
-module load singularity
-SIF=/cluster/tufts/levinlab/hhazan01/singularity/IMAGE.sif
-cd /cluster/tufts/levinlab/hhazan01/PROJECT
-singularity exec --nv --bind /cluster:/cluster "$SIF" \
-    python3 my_script.py --args ...
-```
-
-`--nv` enables GPUs (drop it for CPU-only). `--bind /cluster:/cluster` makes the
-shared filesystem visible inside the container. `--cleanenv` helps when host env
-vars leak in.
-
-## GPU requests (the #1 source of silent OOMs)
-
-**Typed gres does not pin VRAM.** `--gres=gpu:a100:1` can land you on a 40 GB
-A100 *or* an 80 GB A100 — they share the type name `a100`. A 300M+ model that
-needs 80 GB will then OOM at runtime with no obvious cause. **Request by VRAM
-constraint instead:**
-
-```bash
-#SBATCH --gres=gpu:1
-#SBATCH --constraint=a100-80G        # VRAM-precise; see reference file for labels
-```
-
-- Use the VRAM-tagged feature labels (`a100-40G`, `a100-80G`, `h200-141G`,
-  `l40s-48G`, `h100-80G`, …) from the cluster reference file. For a "≥80 GB"
-  job use the alternation `a100-80G|h100-80G|h200-141G`, never coarse
-  `a100|h100|h200` (which silently admits 40 GB cards).
-- **Belt-and-suspenders:** even with a constraint, every A100 (or mixed-VRAM)
-  request must carry the runtime **VRAM guard** from `references/resilience.md`:
-  the runner reads `nvidia-smi` at startup and, if it landed below
-  `MIN_GPU_MEM_MB`, excludes that node and requeues automatically. Constraint +
-  guard together make a wrong-card landing self-correcting.
-- **Constraint fallback ladder** (avoids `Invalid feature specification` hard
-  failures): submit with the full alternation → on reject, retry token-by-token
-  → last resort, retry with no constraint. See `references/resilience.md`.
-
-## Excludes are learned, not hardcoded
-
-**Every submission starts with NO `--exclude`.** Do not bake a stale bad-node
-list into submitters. Instead, the runner discovers a bad node at runtime (wrong
-VRAM, `libtorch_global_deps.so` NFS error, CUDA init failure) and requeues
-itself with that node added to a job-scoped `ExcNodeList`. This way each fresh
-submission re-tests previously-bad nodes (they get fixed), and a still-broken
-node is walked away from within seconds. Mechanism in `references/resilience.md`.
-
-## Splitting big tasks — dependency chains
-
-For work that exceeds one allocation (a multi-stage pipeline, or a run longer
-than the 2d/6d cap split into segments), chain jobs with SLURM dependencies
-rather than one mega-job:
-
-```bash
-jid1=$(sbatch --parsable stage1.sbatch)
-jid2=$(sbatch --parsable --dependency=afterok:$jid1 stage2.sbatch)
-jid3=$(sbatch --parsable --dependency=afterok:$jid2 stage3.sbatch)
-```
-
-`afterok` = run only if the prior succeeded; `afterany` = run regardless (use
-for a resume segment that should start even if the prior hit the walltime);
-`singleton` = serialize same-name jobs. Details and resume-segment patterns in
-`references/resilience.md`.
-
-## Interactive sessions (debugging only, still NOT the login node)
-
-`srun --pty` grabs a compute allocation for quick interactive testing — that is
-a compute node, not the login node. Prefer batch for anything real. Example:
-`srun -p preempt --time=2:00:00 -c 4 --mem=8g --pty bash`.
-
-## Queue ceiling — fine to over-submit
-
-`QOSMaxGRESPerUser` caps *simultaneously running* GPU jobs (~10). Submitting more
-than that is fine — the extras simply sit `PENDING` with reason
-`QOSMaxGRESPerUser` and start as running ones finish. Queue depth is not a
-problem; just submit the whole sweep.
-
-## Preflight checklist before handing over the launch line
-
-- Chosen cluster confirmed (old/new) and matching runtime wrapper used.
-- `--time` within ceiling (2d new / 6d old); default partition time is only
-  15 min, so always set `--time`.
-- Partition exists on that cluster and matches CPU/GPU need.
-- GPU jobs: `--gres=gpu:1` + a **VRAM `--constraint`** (not typed gres) + the
-  runtime VRAM guard for any A100/mixed-VRAM request.
-- No hardcoded `--exclude`; bad nodes are learned at runtime.
-- **Every job**: walltime SIGTERM trap + `--signal=B:USR1@30` + `--requeue` +
-  `--open-mode=append`, auto-requeuing until a `DONE` sentinel (rule 7).
-- **`preempt` jobs MUST carry the full preempt contract (rule 6)**: `--requeue`
-  + `SIGUSR1`/`SIGTERM`/`SIGCONT` trap → `scontrol requeue` + periodic atomic
-  checkpoint + resume-from-last-step. No preempt job without a working
-  save/resume path. Near-cap jobs need the same periodic checkpoint
-  (`references/resilience.md`).
-- `--mem` / `--cpus-per-task` fit a real node in the reference file.
-- Output/data paths point at lab or scratch space, not home.
-- Big/multi-stage work uses dependency chains, not one mega-job.
-- Final answer ends with the single `sbatch` (or dependency/array) line to run,
-  and a note on how to pull results back (`references/intake.md`).
-```
diff --git a/.claude/skills/pax-hpc/references/intake.md b/.claude/skills/pax-hpc/references/intake.md
deleted file mode 100644
index fd741a607..000000000
--- a/.claude/skills/pax-hpc/references/intake.md
+++ /dev/null
@@ -1,92 +0,0 @@
-# Closing the loop — pulling results back and classifying them
-
-The skill's main job ends at a launch line, but a run isn't done until its
-results are back on the user's machine, validated, and filed. This is the return
-half. It has two parts: (1) jobs write **self-describing packages** on the
-cluster, and (2) after the user rsyncs them back, you **classify and
-disseminate** them.
-
-The user runs the rsync. You give the command and do the classification.
-
----
-
-## 1. On the cluster: write a self-describing package per run
-
-A bare directory of `.csv`/`.pt` files is not intake-able — you can't tell a
-finished run from one the walltime killed. Each run's output dir should contain:
-
-- `status.json` — the single source of truth for "did this finish":
-  ```json
-  {"run_id": "myrun_s42", "state": "finished",
-   "exit_code": 0, "started": "...", "ended": "...",
-   "config": {"model": "...", "seed": 42, "lr": 3e-4},
-   "metrics": {"test_ppl": 25.82, "val_ppl": 25.1},
-   "git_commit": "abc1234"}
-  ```
-  `state ∈ {finished, partial, failed}`. Write `partial` from the resilience
-  trap when requeuing, `finished` only on clean completion, `failed` from the
-  EXIT trap on error.
-- `manifest.json` — `{"required_files": ["results.json","history.csv"], "promote_to": "<task>/<variant>"}`.
-  List only what a downstream reader needs; **do not** put resume checkpoints in
-  `required_files` (they're deleted on clean finish).
-- `README.md` — human context: what this run was, how to reproduce
-  (config + commit + seed), node/GPU it ran on.
-
-Have the EXIT trap stamp `status.json` on *every* exit path (success, error,
-OOM-137, requeue) so a killed run is still self-describing.
-
----
-
-## 2. The user rsyncs results back
-
-Pull only the small metadata/metrics, not multi-GB checkpoints:
-
-```bash
-# from the user's local machine — they run this
-rsync -avz --prune-empty-dirs \
-  --include='*/' \
-  --include='status.json' --include='manifest.json' \
-  --include='README.md'   --include='results.json' --include='*.csv' \
-  --exclude='*' \
-  hhazan01@login-p02.pax.tufts.edu:/cluster/tufts/levinlab/hhazan01/PROJECT/out/ \
-  ./from_hpc/PROJECT/
-```
-
-(Use `login-prod-03` for the old cluster.) The `--include/--exclude` filter
-keeps weights on the cluster. If a checkpoint genuinely *is* the deliverable,
-add `--include='*.pt'` deliberately.
-
----
-
-## 3. Classify and disseminate (you do this)
-
-Walk each package under `./from_hpc/` and sort it by `status.json`:
-
-| Classification | Condition | Action |
-|---|---|---|
-| **COMPLETE** | `status.json` valid, `state=finished`, all `required_files` present | Promote to its `promote_to` destination |
-| **INCOMPLETE** | `state=partial`/`failed`, or a required file missing | Offer: rerun missing, rerun all, accept partial, or skip — never silently promote |
-| **UNKNOWN** | `status.json` missing or unparseable | Investigate first; quarantine, do not promote |
-
-- **Promote**: move the package to its final task/variant location; record one
-  log line (what ran, key metric, commit, seed, node/GPU).
-- **Quarantine**: move UNKNOWN/failed-but-kept packages to a
-  `_quarantine/<name>_<timestamp>/` dir so they don't pollute the promoted tree
-  but aren't lost.
-- **Dedup requeued runs**: a resilient job can leave several `run_*` dirs from
-  successive requeues. Keep the one with the highest step / `state=finished`;
-  quarantine the rest. Classify by `(run_id, seed)`, not by directory count, or
-  you double-count requeues.
-- **Audit trail**: append one line per package to an `intake_events.jsonl`
-  (`{ts, package, classification, action, dest}`) so a re-run of intake is
-  idempotent and you can answer "did everything land?" without re-walking.
-
----
-
-## 4. If this is a project with its own intake pipeline
-
-Some projects ship a dedicated intake (validator + promoter + log/STATUS
-regeneration). **Defer to the project's pipeline when one exists** — this file is
-the generic fallback for projects that don't. The contract is the same either
-way: self-describing packages on the cluster → rsync the metadata back →
-classify by `status.json` → promote / quarantine → record what landed.
diff --git a/.claude/skills/pax-hpc/references/new-cluster.md b/.claude/skills/pax-hpc/references/new-cluster.md
deleted file mode 100644
index 3cdad87d3..000000000
--- a/.claude/skills/pax-hpc/references/new-cluster.md
+++ /dev/null
@@ -1,125 +0,0 @@
-# NEW cluster — login-p02.pax.tufts.edu
-
-Modern cluster. Use **conda** to run programs. Apptainer/Singularity 1.4.2 is
-also available at `/usr/bin` if you need a container.
-
-- OS: Rocky Linux 9.6 · Slurm 23.11.11
-- Login: `login-p02.pax.tufts.edu` (the OLD cluster is `login-prod-03`)
-- Compute nodes are named `pax001`–`pax120` (growing; verify the top end with
-  `sinfo`).
-- Runtime: `source ~/.bashrc && conda activate <env>`
-- Envs live in `/cluster/tufts/levinlab/hhazan01/miniconda3/envs/`
-  (verified 2026-05-31): `betse`, `bindsnet`, `delayW`, `dt-d4rl`, `LoRa`.
-  Confirm current list with `conda env list`.
-
-## Scheduler limits (config)
-
-- Max walltime: **2 days** (`2-00:00:00`). Default time if unset: only 15 min —
-  always set `--time`. For runs longer than 2d, use in-place requeue+resume or a
-  dependency chain (see `resilience.md`).
-- Default memory per CPU: 2000 MB. `SelectType=cons_tres`, `CR_CPU_MEMORY`.
-- Max array size: 2000. Max job count: 100000.
-- Your QOS: `interactive`, `normal` (default).
-
-## Partitions
-
-| Partition | Use | Priority | Notes |
-|---|---|---|---|
-| `batch` | default CPU jobs | normal | QoS `normal` (cpu≤250, gpu≤10, mem≤5000G) |
-| `gpu` | lab-allocated GPU jobs | normal | reserved GPU nodes (h100/h200/a100/l40s) — preferred for training |
-| `preempt` | opportunistic, largest pool | low, **requeued when preempted** | QoS `preempt` (cpu≤1000, gpu≤10, mem≤10000G); most nodes incl. many GPUs. **Jobs here MUST be requeue-safe — see `resilience.md`.** |
-
-QOS ceilings: `normal` 2d / cpu=250 / gpu=10 / mem=5000G; `preempt` 2d /
-cpu=1000 / gpu=10 / mem=10000G; `normal-7days` 7d but **no GPU**; `interactive`
-4h / cpu=16 / gpu=1 / mem=64G. `QOSMaxGRESPerUser` caps simultaneously-*running*
-GPU jobs (~10) — over-submitting is fine, extras just queue.
-
-## CPU node types (cores / memory)
-
-- sapphirerapids: 64 cores, ~510 GB (the bulk of `batch`/`preempt`)
-- broadwell: 36–40 cores, 128–257 GB; some 515 GB and 1 TB large-mem
-- Constrain arch with `--constraint=sapphirerapids` or `broadwell` if needed.
-
-## GPUs — request by VRAM label, not bare type
-
-**Do not use typed gres (`--gres=gpu:a100:1`) for memory-sensitive jobs** — the
-type `a100` exists at both 40 GB and 80 GB, so a typed request can land on a
-40 GB card and OOM a 300M+ model. Request generic gres + a **VRAM-precise
-constraint**:
-
-```bash
-#SBATCH --gres=gpu:1
-#SBATCH --constraint=a100-80G       # or an alternation; see tiers below
-```
-
-VRAM-tagged feature labels, GRES type, and where they live (verified
-`sinfo` 2026-05-31):
-
-| GPU | VRAM | Feature label | GRES type | Partition (nodes) |
-|---|---|---|---|---|
-| H200 | 141 GB | `h200-141G` | `h200` | gpu: pax008–011 ·8; preempt: pax047 ·2 (lulab) |
-| H100 | 80 GB | `h100-80G` | `h100` | gpu: pax063 ·3 |
-| A100 | 80 GB | `a100-80G` | `a100` | gpu: pax007, pax049–050, pax105–106 ·8 (icelake) |
-| A100 | 40 GB | `a100-40G` | `a100` | gpu: pax003 ·2, pax051–052 ·8 (cascadelake) |
-| L40S | 48 GB | `l40s-48G` | `l40s` | gpu: pax020–026 ·4; preempt: pax064 ·4 (laolab) |
-| L40 | 48 GB | `l40-48G` | `l40` | preempt: pax048 ·4 (pettilab), pax110–113 ·4 (linlab) |
-| RTX 6000 Ada | 48 GB | `rtx_6000_ada-48G` | `rtx_6000_ada` | preempt: pax062 ·4 (heldweinlab) |
-| RTX A6000 | 48 GB | `rtx_a6000-48G` | `rtx_a6000` | preempt: pax119 ·8 (hugheslab) |
-| RTX A5000 | 24 GB | `rtx_a5000-24G` | `rtx_a5000` | preempt: pax065–068 ·8 (dinglab) |
-| RTX 6000 | 24 GB | `rtx_6000-24G` | `rtx_6000` | preempt: pax118 ·8 (hugheslab) |
-| T4 | 16 GB | `t4-16G` | `t4` | preempt: pax077, pax104 ·4 (linlab) |
-
-- **The type `a100` spans both 40 GB and 80 GB** (different nodes) — this is the
-  concrete reason `--gres=gpu:a100:1` can OOM. Pin with `--constraint=a100-80G`.
-- The `gpu` partition holds the lab/RT-allocated cards (feature `rtgpu`);
-  everything else is lab-private and surfaces only on `preempt` (so it's
-  backfill, and must be requeue-safe — `resilience.md`).
-- Node↔GPU assignments drift and the node count grows; the **labels** are durable,
-  the node names are not. **Never hardcode node names** — query live with
-  `sinfo -N -p gpu,preempt -o '%N %f %G'`.
-- The libtorch-NFS breakage that once hit pax009/010/011/024/025/026/063 is
-  cleared as of 2026-05-31 (all healthy with GRES) — exactly why excludes are
-  learned at runtime (`resilience.md` §4), never baked in.
-
-### Tier helper (drop into a submitter)
-
-```bash
-constraint_for_tier() {
-    case "$1" in
-        flex)      echo "" ;;                                                  # any GPU
-        ge_24g)    echo "rtx_a5000-24G|l40-48G|l40s-48G|rtx_6000_ada-48G|a100-40G|a100-80G|h100-80G|h200-141G" ;;
-        ge_40g)    echo "a100-40G|a100-80G|h100-80G|h200-141G|l40-48G|l40s-48G|rtx_6000_ada-48G" ;;
-        ge_48g)    echo "l40-48G|l40s-48G|rtx_6000_ada-48G|a100-80G|h100-80G|h200-141G" ;;
-        ge_80g)    echo "a100-80G|h100-80G|h200-141G" ;;                       # ≥80 GB, no 40G slip
-        a100_80)   echo "a100-80G" ;;                                          # avoid H100/H200 contention
-        h200)      echo "h200-141G" ;;
-        h100_h200) echo "h100-80G|h200-141G" ;;
-        *)         echo "" ;;
-    esac
-}
-```
-
-Tier guidance:
-- 300M body, dense or LoRA r≥256 → **`ge_80g`** (the 40 GB card OOMs).
-- 600M+ FFN-trainable / dense ablations → **`h200`** (80 GB cards peaked ~76 GB
-  and OOMed in past runs).
-- 2B body → **`h200`**, no exceptions.
-- Tiny pilots (≤10M) → **`flex`**, or `ge_24g` if the runner needs ≥24 GB.
-
-Always pair the constraint with the runtime **VRAM guard** (`resilience.md` §3)
-and set `MIN_GPU_MEM_MB` to match — the guard requeues off a wrong-VRAM landing
-instead of silently OOMing.
-
-Example GPU header:
-
-```bash
-#SBATCH --partition=gpu
-#SBATCH --gres=gpu:1
-#SBATCH --constraint=a100-80G
-#SBATCH --cpus-per-task=8
-#SBATCH --mem=64G
-#SBATCH --time=01-00:00:00
-```
-
-> The user's old `.bashrc` GPU aliases reference `p100`/`k20xm` — those are
-> OLD-cluster GPUs and do not exist here. Pick from the table above.
diff --git a/.claude/skills/pax-hpc/references/old-cluster.md b/.claude/skills/pax-hpc/references/old-cluster.md
deleted file mode 100644
index 121abfb3a..000000000
--- a/.claude/skills/pax-hpc/references/old-cluster.md
+++ /dev/null
@@ -1,105 +0,0 @@
-# OLD cluster — login-prod-03.pax.tufts.edu
-
-Legacy cluster. **Use Singularity to run programs — always, by user policy.**
-conda fails on the cluster's `rhel7` nodes (glibc < 2.28 → `GLIBC_2.28 not
-found`), which are the majority of the pool and **include every A100**. The
-`rhel8` nodes (l40 / h100 / l40s / rtx_a5000, on `preempt`) *could* in principle
-run conda via `-C rhel8`, but that would (a) lose all A100 access and (b) split
-the pool into two runtimes. **The user deliberately uses Singularity for ALL
-old-cluster jobs to avoid that complication — do not propose conda here.**
-Containerize the workload.
-
-- OS: RHEL 7.5 (Maipo) · Slurm 23.02.7
-- Login: `login-prod-0X.pax.tufts.edu` (there are several — e.g. `login-prod-01`,
-  `login-prod-03`).
-- Compute nodes keep the **legacy** naming `cc1gpu###`, `s1cmp###`, `p1cmp###`,
-  `d1cmp###` — this cluster did **not** migrate to `pax###` (only the NEW cluster
-  did). Don't expect `pax` names here.
-- Runtime: `module load singularity` then `singularity exec ...`
-  (apptainer/singularity is a module, not on the default PATH)
-- Images live at `/cluster/tufts/levinlab/hhazan01/singularity/*.sif`
-  (e.g. `mettagrid.sif`, `delayW.sif`, `LoRa.sif`, `Heb.sif` — confirm with `ls`)
-
-## Run wrapper
-
-```bash
-module load singularity
-SIF=/cluster/tufts/levinlab/hhazan01/singularity/IMAGE.sif
-singularity exec --nv --bind /cluster:/cluster "$SIF" python3 script.py --args
-```
-
-- `--nv` = expose GPUs (omit for CPU-only jobs)
-- `--bind /cluster:/cluster` = make the shared filesystem visible in-container
-- `--cleanenv` is useful when host env vars leak into the container
-
-## Scheduler limits (config)
-
-- Max walltime: **6 days** (`6-00:00:00`) — longer than the new cluster.
-- Default memory per CPU: 2000 MB. `cons_tres`, `CR_CPU_MEMORY`.
-- Max array size: 2000. Max job count: 100000.
-
-## Partitions
-
-| Partition | Use | Walltime | Notes |
-|---|---|---|---|
-| `batch` | default CPU jobs | 6d | rhel7/rhel8 nodes |
-| `gpu` | GPU jobs | 6d | a100/p100 reserved GPU nodes |
-| `mpi` | multi-node MPI | 6d | large core counts |
-| `largemem` | big-memory jobs | 6d | ~1 TB nodes |
-| `interactive` | quick interactive | 4h | for `srun --pty` |
-| `preempt` | opportunistic, largest/most varied pool | 6d | **requeued when preempted**; widest GPU selection. **Jobs here MUST be requeue-safe — see `resilience.md`.** |
-
-QOS ceilings: `preempt` cpu≤4000 / gpu≤20 / mem≤10000G; `public` cpu≤2000 /
-gpu≤10 / mem≤5000G; `expanded` cpu≤2000 / gpu≤24; `normal` default.
-
-## CPU node types
-
-- rhel8 nodes: 128 cores, ~510 GB (newest of the old pool)
-- broadwell rhel7: 36+ cores, 120–248 GB; large-mem up to ~1 TB / 1.5 TB
-- Constrain OS generation if a container needs it: `--constraint=rhel8` or
-  `--constraint=rhel7`.
-
-## GPUs
-
-Request generic gres + a VRAM/type constraint (same rationale as the new
-cluster — typed `--gres=gpu:a100:1` does not pin VRAM):
-
-```bash
-#SBATCH --partition=gpu        # or preempt
-#SBATCH --gres=gpu:1
-#SBATCH --constraint=a100      # tighten to a VRAM label if exposed (sinfo -o '%N %f %G')
-```
-
-Verified `sinfo` 2026-05-31. **VRAM feature labels exist only for A100 here** —
-everything else must be requested by GRES type, and a few labels are inconsistent
-(noted below):
-
-| GPU | VRAM | Feature label | GRES type | Nodes (count) |
-|---|---|---|---|---|
-| A100 | 40 GB | `a100-40G` ✓ | `a100` | cc1gpu001–005 ·8; p1cmp110–111 ·2 |
-| A100 | 80 GB | `a100-80G` ✓ | `a100` | s1cmp001–005 ·8; s1cmp006–007 ·2 |
-| H100 | 80 GB | **none** ⚠️ | `h100` | s1cmp010 ·3 — **no feature label; request by GRES only** |
-| RTX A6000 | 48 GB | `rtx_a6000` | `rtx_a6000` | s1cmp008 ·8 |
-| RTX 6000 Ada | 48 GB | `rtx_a6000ada` ⚠️ | `rtx_6000ada` ⚠️ | s1cmp009 ·4 — **feature/GRES names disagree** |
-| RTX A5000 | 24 GB | `rtx_a5000` | `rtx_a5000` | s1cmp012–015 ·8 |
-| L40 | 48 GB | `l40` | `l40` | d1cmp042–050 ·4 |
-| L40S | 48 GB | `l40` ⚠️ | `l40s` | s1cmp011 ·4 — **feature says `l40` but GRES is `l40s`** |
-| RTX 6000 | — | `rtx_6000` | `rtx_6000` | p1cmp077 ·2 |
-| V100 | — | `v100` | `v100` | p1cmp071–076 (·2–4) |
-| P100 | — | `p100` | `p100` | p1cmp073 ·4, p1cmp075 ·6 |
-| T4 | 16 GB | `t4` (no VRAM tag) | `t4` | p1cmp090–109 ·4 (≈20 nodes) |
-
-Practical rules for the old cluster:
-- **A100**: use `--constraint=a100-80G` / `a100-40G` (labels are reliable). Pair
-  with the runtime VRAM guard (`resilience.md` §3).
-- **H100**: there is no feature label — request `--gres=gpu:h100:1` directly.
-- **L40S vs L40**: don't trust the `l40` feature to mean 48 GB L40 specifically;
-  if you need L40S, request `--gres=gpu:l40s:1`. If you need L40, `--gres=gpu:l40:1`.
-- **RTX 6000 Ada**: feature is `rtx_a6000ada` but GRES is `rtx_6000ada` — use the
-  **GRES** form `--gres=gpu:rtx_6000ada:1`.
-- Always confirm live: `sinfo -N -p gpu,preempt -o '%N %f %G'`. Node names and
-  labels here are messier than the new cluster — never hardcode them.
-
-The user's `.bashrc` interactive GPU aliases (`sbash_gpu` → `gpu:p100:1`,
-`sbash_gpu_k20_x` → `gpu:k20xm:1` on partition `gpu`) were written for this
-cluster.
diff --git a/.claude/skills/pax-hpc/references/resilience.md b/.claude/skills/pax-hpc/references/resilience.md
deleted file mode 100644
index 00c8f2af2..000000000
--- a/.claude/skills/pax-hpc/references/resilience.md
+++ /dev/null
@@ -1,457 +0,0 @@
-# Resilience — requeue, resume, preemption, VRAM guard, dependencies
-
-Any job that can be interrupted must survive the interruption: catch the kill,
-save a checkpoint, requeue itself, and resume from that checkpoint on restart.
-
-**The walltime trap is universal (user directive, 2026-06-10): EVERY job — not
-just preempt or near-cap jobs — traps the SIGTERM the scheduler sends at the
-`--time` limit and `scontrol requeue`s itself, auto-requeuing until the run
-writes a `DONE` sentinel.** Walltime estimates are routinely wrong; a job that
-overruns must resume, never silently die. The *periodic checkpoint* layer
-(§0.2) stays scoped to **every `preempt` job** and **every job whose walltime
-approaches the cluster cap** (2 days new / 6 days old) — that's what lets a
-requeued job pick up near where it died instead of from step 0.
-
-This file gives copy-paste scaffolding. Drop the bash blocks into the runner
-(the script that executes on the compute node), not the submitter.
-
----
-
-## 0. The signal facts (read first)
-
-Verified on the NEW cluster 2026-05-31: `KillWait=30 sec`,
-`PreemptMode=REQUEUE`, `PreemptType=preempt/partition_prio`,
-`PreemptExemptTime=0`.
-
-- **Signal 9 (SIGKILL) cannot be caught, trapped, or ignored.** Anything that
-  says "catch kill -9" is impossible. You catch the signal SLURM sends *before*
-  the SIGKILL, during a grace window.
-- On **preemption** (`preempt` partition), SLURM sends `SIGCONT` + `SIGTERM`,
-  waits the partition `GraceTime`, then `SIGKILL`. **`GraceTime=0` on the preempt
-  partition of BOTH clusters (verified 2026-05-31)** — so on preemption there is
-  **no on-signal window at all**; SIGTERM and SIGKILL arrive back-to-back and the
-  handler cannot complete a checkpoint. For preempt jobs the **periodic
-  checkpoint (every K steps) is the ONLY safety net** — this is mandatory, not
-  optional (see §0.2). The trap is dead weight on preemption; it earns its keep
-  only on the walltime path below.
-- On **time limit**, SLURM sends `SIGTERM` at the limit, waits `KillWait` (=30 s
-  here), then `SIGKILL`. To get advance warning *before* the limit, request a
-  signal: `#SBATCH --signal=B:USR1@30` → SLURM delivers `SIGUSR1` to the batch
-  shell ~30 s before the limit. (`B:` = batch shell, not the job steps. `@30`
-  matches the 30 s `KillWait`; raise it only if a checkpoint write needs longer.)
-- So the robust runner traps **SIGUSR1 + SIGTERM + SIGCONT** and routes
-  all three to the same "checkpoint + requeue" handler. This covers preemption
-  *and* walltime on both clusters with one code path.
-
-Required SBATCH directives for any resilient job:
-
-```bash
-#SBATCH --signal=B:USR1@30     # advance warning before the time limit
-#SBATCH --open-mode=append     # preserve logs across requeues (do NOT overwrite)
-#SBATCH --requeue              # mark job requeue-ELIGIBLE (necessary, not sufficient — see §0.1)
-```
-
-## 0.1 Critical: `PreemptMode=REQUEUE` does NOT requeue your job by itself
-
-Verified empirically on this cluster: jobs preempted on `preempt` are **not**
-auto-requeued, despite `PreemptMode=REQUEUE`. Two things defeat the automatic
-path, and you must assume both:
-
-1. **The job must be requeue-eligible.** Without `#SBATCH --requeue` the job is
-   not requeueable, so `PreemptMode=REQUEUE` has nothing to requeue and the job
-   is **cancelled** on preemption. With `GraceTime=0` this is the most likely
-   reason a preempted job vanishes instead of requeuing — it never got a chance
-   to do anything. **`--requeue` is mandatory.**
-2. **A clean exit cancels the requeue** (the walltime path). If your handler ends
-   with `exit 0` *before* SLURM kills the job — which only happens when there IS
-   a grace window, i.e. at the time limit (`KillWait=30`), not on `GraceTime=0`
-   preemption — SLURM records COMPLETED and won't requeue. So at the walltime
-   boundary you must call `scontrol requeue` *explicitly* in the handler.
-3. **Auto-requeue only re-runs the batch script from the top** — it never
-   restores program state. Resume is your code's job regardless (§2), and on
-   preemption the periodic checkpoint (§0.2) is the only thing that survived.
-
-**The fix (what to propose): requeue yourself, explicitly, from the batch
-script — do not depend on `PreemptMode`.** In the trap handler call
-`scontrol requeue "$SLURM_JOB_ID"` *before* the job can exit. Once that call is
-accepted the job is marked for requeue and will restart even if SIGKILL hits a
-moment later — which is why we requeue *first* and checkpoint *second* (a slow
-checkpoint must never cost you the requeue). `#SBATCH --requeue` only makes the
-job *eligible*; the explicit `scontrol requeue` is what actually does it.
-
-**Requeue from bash, checkpoint from Python.** Put the `scontrol requeue` in the
-SLURM batch (bash) layer, not in the training code — bash is the job's top
-process, it can requeue even if Python is wedged, and it keeps the training
-script SLURM-agnostic. Python's only job is: trap `SIGUSR1` → write the latest
-checkpoint → exit. This division is the "whatever is safer" answer: the requeue
-can't be lost to a hung trainer, and the trainer stays portable.
-
-## 0.2 MANDATORY: periodic checkpointing for every preempt job
-
-Because `GraceTime=0`, a preempted job is killed with no warning the handler can
-act on. The **only** state that survives is the last checkpoint the training loop
-wrote on its own schedule. Therefore, for any job on `preempt` (or any job that
-could be requeued), periodic checkpointing is **mandatory**:
-
-- Write a checkpoint every `K` steps such that **at most ~30 min of compute** is
-  ever at risk (tune `K` to the model's step time). This bounds worst-case loss
-  on an unannounced preemption.
-- Overwrite a single latest checkpoint with an **atomic write** (write to
-  `*.tmp`, then `mv` into place) so a kill mid-write can't corrupt the resume
-  point.
-- Delete the checkpoint on clean finish so a completed run leaves no resume bait.
-- The `SIGUSR1` handler's checkpoint is a *best-effort bonus* for the walltime
-  path only; never rely on it for preemption. A preempt job without periodic
-  checkpointing is not resilient — it just hasn't been preempted yet.
-
-Combined with `#SBATCH --requeue`, this is the whole preempt-survival contract:
-**`--requeue` (so SLURM puts it back) + periodic checkpoint (so it resumes near
-where it died)**. The trap adds the walltime case on top.
-
----
-
-## 0.3 Sizing the grace window AND the walltime (the 2026-06-10 timeout incident)
-
-Two sizing mistakes both end in a dead job; the trap only saves you from the first.
-
-**Grace (`@N` in `--signal=B:USR1@N`) must cover the time to the NEXT checkpoint
-opportunity, not just the checkpoint write.** If the loop can only checkpoint at a
-step/epoch boundary, and a boundary is preceded by a long *uninterruptible* section
-— an eval pass, a validation epoch, a big batch — then `@30` is too short: the
-handler sets the flag, but the flag is not read until that section finishes, and
-SIGKILL lands first. Size `N` ≥ (longest uninterruptible section + checkpoint
-write). Example: the DT-AntMaze runner evals between train steps, so it uses
-`--signal=B:USR1@300`, not `@30`.
-
-**Walltime must be sized to the SLOWEST cell of a sweep — never inherited from a
-faster one.** Incident (2026-06-10): a Meta-LoRA DT sweep set `--time` per *cadence*
-(mild 3:00, medium 3:30, aggressive 5:00) assuming more-frequent resampling = slower.
-But runtime was dominated by the *dataset*: the `umaze-diverse` runs took ~3.0–4.5 h
-regardless of cadence. The 3:00 / 3:30 limits — fine for the ~2× faster non-diverse
-`umaze` grid they were copied from — were ~5 min too short, and **44 of 90 jobs hit
-TIMEOUT** while the 5:00 aggressive cells (same dataset) all finished. The kicker:
-the trainer already had a full SIGUSR1 → checkpoint → resume path, but the SLURM
-template never set `--signal` / `--requeue`, so the trap never fired and the work
-was lost instead of resumed. That is exactly the failure rules 6–7 exist to prevent
-— wire the trap and a too-short walltime becomes a free requeue, not 44 dead jobs.
-
----
-
-## 1. The trap + checkpoint + requeue handler (CRASH-SAFE walltime default)
-
-**Requeue IF AND ONLY IF: (a) the walltime warning fired AND (b) the run did not
-finish.** Never on a crash, never on a `scancel`, never on a job that completed
-before walltime. User directive (2026-06-10/-12): auto-requeue an overrunning job
-until it finishes, but never resurrect a job that died for another reason — that
-loops a crash forever or fights an operator's `scancel`.
-
-The two discriminators (note: **the exit code is NOT one** — see below):
-
-- **`SIGUSR1` = the walltime advance-warning** from `--signal=B:USR1@N`. ONLY a
-  walltime stop produces it; a crash, a `scancel`, and a `GraceTime=0` preemption do
-  not. So `WALLTIME=1` (set by the USR1 trap) is the authoritative "this is a
-  walltime stop" flag, and it is what excludes crash and cancel.
-- **A completion sentinel** written ONLY on true finish (`metrics.json`, `DONE`, …):
-  present ⇒ finished ⇒ no requeue. This excludes "completed before walltime."
-- **Trap `USR1` + `TERM` + `CONT`.** USR1 sets `WALLTIME=1` and forwards a checkpoint
-  request to the child. TERM/CONT also forward a checkpoint and — by being trapped —
-  keep the shell alive through the hard-kill window so the post-`wait` decision runs
-  instead of the shell dying instantly. A `scancel` is also `TERM`, but it arrives
-  with `WALLTIME=0`, so it is never requeued.
-- **Do NOT gate the requeue on the exit code.** A walltime stop can exit cleanly
-  (the trainer checkpointed and exited 0) *or* be hard-killed non-zero (its checkpoint
-  ran past the grace). Both must requeue. `WALLTIME=1` already excludes crash and
-  cancel, so testing `rc` adds nothing and would wrongly drop the hard-killed case.
-  (For a *generic* job that does NOT self-handle USR1, the equivalent test is "exited
-  non-zero after the walltime warline" — i.e. requeue iff `WALLTIME=1 && rc!=0` and
-  there is no sentinel; a clean `rc==0` then means it completed. Our trainers write a
-  sentinel, so we key on the sentinel, which is unambiguous either way.)
-- `SLURM_RESTART_COUNT` hard-caps the loop (a backstop, not the stop condition).
-
-```bash
-# ---- resilience: crash-safe walltime requeue -----------------------------
-REQUEUE_CAP=8
-WALLTIME=0
-CHILD_PID=""
-SENTINEL="${RUN_DIR}/metrics.json"          # written ONLY on true completion
-
-_ckpt_child() { [ -n "$CHILD_PID" ] && kill -0 "$CHILD_PID" 2>/dev/null && kill -USR1 "$CHILD_PID" 2>/dev/null || true; }
-_on_usr1() { WALLTIME=1; echo "[resilience] SIGUSR1 (walltime) $(date -Is); child -> checkpoint+exit"; _ckpt_child; }
-_on_term() { echo "[resilience] TERM/CONT (walltime=$WALLTIME); forward checkpoint, defer requeue decision"; _ckpt_child; }
-trap _on_usr1 USR1
-trap _on_term TERM CONT                      # keep the shell alive; scancel(WALLTIME=0) won't requeue
-
-python my_script.py --resume-dir "$RUN_DIR" --args ... &
-CHILD_PID=$!
-# wait returns >128 either because a trap interrupted it (child alive — re-wait) OR
-# because the child itself died from a signal (already reaped). Re-wait ONLY while the
-# child is still alive, so a signal-death (SIGKILL/SIGSEGV) cannot spin the loop.
-rc=0; wait "$CHILD_PID" || rc=$?
-while [ "$rc" -gt 128 ] && kill -0 "$CHILD_PID" 2>/dev/null; do rc=0; wait "$CHILD_PID" || rc=$?; done
-
-if [ -f "$SENTINEL" ]; then
-    echo "[resilience] finished before walltime (sentinel, rc=$rc) — no requeue"
-elif [ "$WALLTIME" -eq 1 ]; then            # walltime + unfinished — rc-independent
-    n="${SLURM_RESTART_COUNT:-0}"
-    if [ "$n" -lt "$REQUEUE_CAP" ]; then
-        echo "[resilience] walltime + unfinished (rc=$rc, $n/$REQUEUE_CAP) — requeue"
-        scontrol requeue "$SLURM_JOB_ID"; exit 0
-    fi
-    echo "[resilience] requeue cap reached — FAIL" >&2; exit 1
-else
-    echo "[resilience] rc=$rc walltime=0 no sentinel — crash/cancel/preempt, NO requeue" >&2
-    exit "$([ "$rc" -eq 0 ] && echo 1 || echo "$rc")"
-fi
-# --------------------------------------------------------------------------
-```
-
-The requeue decision is made *after* the child is reaped (not "requeue first") so it
-can see the sentinel and the `WALLTIME` flag together. The slow-checkpoint risk that
-motivates "requeue first" is covered instead by a generous grace (`@300`, §0.3) plus
-trapping `TERM` (which holds the shell open if the hard kill arrives before the child
-finishes). For `GraceTime=0` **preemption** there is no warning at all — see §1b.
-
-## 1b. Preemption addendum (only for `preempt`-partition jobs)
-
-Preemption delivers `SIGTERM`+`SIGKILL` with `GraceTime=0` — no warning, and the
-`SIGTERM` is indistinguishable from a `scancel`. So the §1 walltime handler cannot
-safely requeue a preemption. For preempt jobs the survival contract is instead
-**`#SBATCH --requeue` (eligibility) + periodic checkpoint (§0.2) + resume (§2)**:
-SLURM's `PreemptMode=REQUEUE` puts the eligible job back, and it resumes from the last
-periodic checkpoint. Do not add a `SIGTERM` requeue trap to "improve" this — it buys
-nothing on `GraceTime=0` and reintroduces the scancel-requeue bug.
-
-**Python side** (the child) traps `SIGUSR1`, writes the latest checkpoint, and
-exits non-zero-but-clean (or 0) so the bash handler proceeds:
-
-```python
-import signal, sys
-_should_stop = False
-def _on_usr1(signum, frame):
-    global _should_stop
-    _should_stop = True
-signal.signal(signal.SIGUSR1, _on_usr1)
-# ... in the training loop, after each step/epoch:
-if _should_stop:
-    save_checkpoint(run_dir)   # latest only; overwrite previous
-    sys.exit(0)
-```
-
-If your training framework can't be interrupted mid-step, the simplest robust
-form is: checkpoint every K steps unconditionally, and on `SIGUSR1` just set the
-flag and let the next step boundary save+exit. The grace window only needs to
-cover one checkpoint write.
-
----
-
-## 2. Resume on restart (robust, sentinel-independent)
-
-When the requeued job starts, it must find the latest checkpoint and resume.
-**Do not rely on a single `last_run_dir.txt` sentinel** — that has silently
-gone stale after multiple requeues on this cluster, causing a fresh-from-zero
-restart that abandons a near-finished run. Probe the filesystem directly for the
-highest-step checkpoint:
-
-```bash
-# Prefer an explicit override; else find the newest checkpoint by step number.
-RUN_DIR="${RESUME_RUN_DIR:-}"
-if [ -z "$RUN_DIR" ]; then
-    latest_ckpt=$(ls -1 "$OUT_ROOT"/run_*/checkpoints*/checkpoint_*.pt 2>/dev/null \
-        | sed -E 's/.*checkpoint_[a-z]*([0-9]+)\.pt/\1 &/' \
-        | sort -n | tail -1 | cut -d' ' -f2-)
-    [ -n "$latest_ckpt" ] && RUN_DIR=$(dirname "$(dirname "$latest_ckpt")")
-fi
-if [ -n "$RUN_DIR" ] && [ -d "$RUN_DIR" ]; then
-    echo "[resilience] resuming from $RUN_DIR"
-else
-    RUN_DIR="$OUT_ROOT/run_$(date +%s)"; mkdir -p "$RUN_DIR"
-    echo "[resilience] no checkpoint found; starting fresh in $RUN_DIR"
-fi
-```
-
-- Always honor a manual `RESUME_RUN_DIR=<dir>` env override — it's the escape
-  hatch when automatic detection picks wrong.
-- Keep only the latest checkpoint (overwrite), and **delete it on clean finish**
-  so a finished run doesn't leave resume bait or waste disk.
-- `--open-mode=append` (set in the submitter) keeps every requeue's log in one
-  file so the failure trail survives.
-
----
-
-## 3. VRAM guard — auto-requeue off a too-small GPU
-
-Even with a VRAM `--constraint`, add this backstop for any A100 / mixed-VRAM
-request. At job start the runner checks the card it actually got; if it's below
-the requirement it excludes that node and requeues. Over repeated requeues the
-job walks itself onto a correct card.
-
-```bash
-MIN_GPU_MEM_MB="${MIN_GPU_MEM_MB:-80000}"   # set to the model's real need
-gpu_mem=$(nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits 2>/dev/null | head -1)
-if [ -n "$gpu_mem" ] && [ "$gpu_mem" -lt "$MIN_GPU_MEM_MB" ]; then
-    echo "[vram-guard] landed on ${gpu_mem} MiB < required ${MIN_GPU_MEM_MB} on ${SLURMD_NODENAME}; excluding + requeuing"
-    _exclude_node_and_requeue "$SLURMD_NODENAME"   # see §4
-    exit 0
-fi
-echo "[vram-guard] GPU OK: ${gpu_mem} MiB on ${SLURMD_NODENAME}"
-```
-
-Set `MIN_GPU_MEM_MB` to match the constraint — don't claim `a100-80G` but set
-`MIN_GPU_MEM_MB=40000`, or the guard goes slack and you get confusing late OOMs.
-
----
-
-## 4. Adaptive exclude — start empty, learn at runtime
-
-**Submitters set NO `--exclude`.** Each submission re-tests every node (broken
-nodes do get fixed). The runner discovers a bad node — wrong VRAM (§3), a
-`libtorch_global_deps.so: cannot open shared object file` NFS error, a CUDA init
-failure — and requeues itself with that node appended to a **job-scoped**
-exclude list via `ExcNodeList`:
-
-```bash
-_exclude_node_and_requeue() {
-    local bad="$1"
-    # Accumulate across this job's requeues in a small per-job file.
-    local exfile="${EXCLUDE_FILE:-$OUT_ROOT/.exclude_${SLURM_JOB_ID}}"
-    echo "$bad" >> "$exfile"
-    local list; list=$(sort -u "$exfile" | paste -sd, -)
-    echo "[exclude] requeuing ${SLURM_JOB_ID} with ExcNodeList=${list}"
-    scontrol requeue "$SLURM_JOB_ID"
-    scontrol update JobId="$SLURM_JOB_ID" ExcNodeList="$list"
-}
-```
-
-This is strictly better than a hardcoded list: it's current (re-tests on every
-fresh submission), self-healing (walks off a broken node in seconds), and leaves
-no stale exclusions to silently shrink the pool months later.
-
-Guard the runner's startup with a quick sanity import so a node-level breakage is
-caught *before* training starts:
-
-```bash
-python -c "import torch; assert torch.cuda.is_available()" \
-    || _exclude_node_and_requeue "$SLURMD_NODENAME"
-```
-
----
-
-## 5. Walltime self-requeue (universal — until the job is DONE)
-
-**This is mandatory on every job, regardless of expected runtime** (rule 7).
-The same trap handles the cap: `--signal=B:USR1@30` fires ~30 s before the
-`--time` limit, the handler checkpoints + `scontrol requeue`s, and the requeued
-job resumes (§2). So a run that overruns one allocation just needs `--time` set,
-the trap wired, and a working resume — it will tick across as many requeues as
-needed.
-
-**"Until completed" needs a stop condition.** Auto-requeue must end when the
-work is finished, not loop forever after a clean finish or a non-walltime crash.
-Gate it on a `DONE` sentinel the training code writes on successful completion:
-resume while `DONE` is absent, and skip the trap/requeue once it exists.
-
-```bash
-# at the top of the runner, before launching training:
-if [ -f "$RUN_DIR/DONE" ]; then
-    echo "[resilience] $RUN_DIR already DONE; nothing to requeue"; exit 0
-fi
-# ... trap _on_signal USR1 TERM CONT ; run training ...
-# the trainer writes "$RUN_DIR/DONE" (atomically) only on successful completion.
-# the _on_signal handler should no-op if DONE already exists:
-#   _on_signal() { [ -f "$RUN_DIR/DONE" ] && exit 0; ... ; scontrol requeue ...; }
-```
-
-Verify the checkpoint cadence is short enough that losing the work since the
-last checkpoint is acceptable (e.g. checkpoint every ≤30 min of compute).
-
-For the **new cluster's 2-day cap specifically**: set `--time=2-00:00:00`, wire
-the trap, and the job auto-requeues + resumes at the boundary. (There is a
-`normal-7days` QOS on the new cluster but it has **no GPU** — it can't substitute
-for resume on GPU work.)
-
----
-
-## 6. Splitting big tasks — dependency chains
-
-When a task is naturally staged, or you'd rather split a long run into bounded
-segments than rely on in-place requeue, chain jobs with `--dependency`:
-
-```bash
-jid1=$(sbatch --parsable stage1.sbatch)                                  # e.g. preprocess
-jid2=$(sbatch --parsable --dependency=afterok:$jid1   stage2.sbatch)     # train, runs iff stage1 ok
-jid3=$(sbatch --parsable --dependency=afterok:$jid2   stage3.sbatch)     # eval/package
-```
-
-Dependency types:
-- `afterok:JID` — start only if JID succeeded (exit 0).
-- `afterany:JID` — start regardless of how JID ended. Use for a **resume
-  segment**: segment N+1 should run even if segment N hit the walltime (non-zero
-  exit), picking up from the checkpoint.
-- `afternotok:JID` — start only if JID failed (cleanup / fallback paths).
-- `singleton` — serialize all jobs sharing the same `--job-name` + user; a clean
-  way to make a self-chaining resume job never run two copies at once.
-
-Self-chaining segment pattern (each segment trains for ~1 cap, then submits its
-own successor before exiting, conditioned on "not yet converged"):
-
-```bash
-# near the end of the runner, if training is incomplete:
-if [ ! -f "$RUN_DIR/DONE" ]; then
-    sbatch --parsable --dependency=afterany:${SLURM_JOB_ID} "$0"   # queue the next segment
-fi
-```
-
-Prefer in-place requeue (§1–§2) for a single long run; prefer dependency chains
-when stages have distinct resource shapes (CPU preprocess → GPU train → CPU
-package) or when you want each segment's logs and accounting separated.
-
----
-
-## 7. Putting it together — minimal resilient GPU runner skeleton
-
-```bash
-#!/usr/bin/env bash
-#SBATCH --job-name=myrun
-#SBATCH --output=./res_%j_%N.txt
-#SBATCH --error=./res_%j_%N.err
-#SBATCH --time=2-00:00:00
-#SBATCH --partition=preempt
-#SBATCH --gres=gpu:1
-#SBATCH --constraint=a100-80G
-#SBATCH --cpus-per-task=8
-#SBATCH --mem=64G
-#SBATCH --signal=B:USR1@30
-#SBATCH --open-mode=append
-#SBATCH --requeue
-
-set -uo pipefail
-OUT_ROOT=/cluster/tufts/levinlab/hhazan01/PROJECT/out
-MIN_GPU_MEM_MB=80000
-
-source ~/.bashrc && conda activate LoRa          # NEW cluster
-# (OLD cluster: module load singularity; wrap the python line in singularity exec)
-
-# --- instrumentation: which node/GPU did we actually get? (see ../SKILL.md) ---
-echo "===== START $(date -Is)  node=${SLURMD_NODENAME} job=${SLURM_JOB_ID} ====="
-nvidia-smi --query-gpu=name,memory.total --format=csv,noheader || true
-
-# --- §4 startup sanity + §3 VRAM guard ---
-# ( _exclude_node_and_requeue + _on_signal defined as above )
-python -c "import torch; assert torch.cuda.is_available()" || _exclude_node_and_requeue "$SLURMD_NODENAME"
-gpu_mem=$(nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits | head -1)
-[ "${gpu_mem:-0}" -lt "$MIN_GPU_MEM_MB" ] && { _exclude_node_and_requeue "$SLURMD_NODENAME"; exit 0; }
-
-# --- §2 resolve resume dir ---
-# ( RUN_DIR resolution block as above )
-
-# --- §1 trap + launch ---
-trap _on_signal USR1 TERM CONT
-python my_script.py --resume-dir "$RUN_DIR" --args ... &
-CHILD_PID=$!
-wait "$CHILD_PID"
-echo "===== END $(date -Is)  exit=$? ====="
-```
-
-This single runner is preempt-safe, walltime-safe, lands on a correct-VRAM GPU
-or requeues, learns bad nodes with no hardcoded exclude, and resumes from its own
-checkpoint — the full set of guarantees the user asked for.
diff --git a/.claude/skills/report/SKILL.md b/.claude/skills/report/SKILL.md
deleted file mode 100644
index f1afd0c53..000000000
--- a/.claude/skills/report/SKILL.md
+++ /dev/null
@@ -1,53 +0,0 @@
----
-name: report
-description: Plain-English status/results report for any project. Use whenever the user
-  asks for "a report", "what ran", "status", "what happened", "summary", "explain the
-  results", or wants work explained simply for a non-specialist. Reads the project's own
-  durable records (logs, status files, trackers, git history).
----
-
-# Plain-English report
-
-The reader is smart and busy, wants the truth fast, and hates jargon. Write so a
-non-specialist on this project can follow it.
-
-## Rules
-- Plain English. Short sentences, common words. If a technical term is unavoidable,
-  define it in the same breath, once (e.g. "gated = run through a strict pass/fail test").
-- Facts only. Every claim ties to a source: a run/commit id, a number, a file, a ticket.
-  Never invent. If something is unknown, say "unknown".
-- No spin. Negative, null, and failed results get equal weight. If an earlier claim was
-  reversed, say so plainly under Corrections.
-- Brief. Group many items by theme; headline per theme, not a play-by-play.
-
-## Figures & diagrams (a picture often beats a paragraph)
-When a figure makes the report clearer, include one — and add a one-line caption saying
-the single thing it shows:
-- Concept / architecture / flow / decision -> an ASCII diagram inline.
-- Data / results (trends, comparisons, distributions) -> generate a REAL plot from the
-  data with a small script (never hand-draw numbers); save it under the project's figures
-  area and reference it. Follow any "figures regenerated from data" rule the project has.
-- Prefer a figure over a long paragraph when it compresses the idea. Skip decorative ones;
-  every figure must earn its place.
-
-## Find the sources first (use whatever the project actually keeps)
-experiment logs, LOG/STATE/STATUS/CHANGELOG/NOTES, lab notebooks; trackers / results
-tables (csv, json, dashboards); recent git commits, open PRs/issues, CI; README/docs for
-baseline and goal. For large reads use a subagent; bring back only conclusions. If there
-is no durable record, say so and report from what's observable.
-
-## Format
-One opening line: the big-picture arc.
-Then per theme (or per item if asked):
-- Task — what was tried, one sentence.
-- Baseline — the number/behaviour being compared against.
-- Result — real numbers (value ± spread, sample size, conditions); a figure if it helps.
-- What it means — the takeaway, plain.
-- How we know — why it's trustworthy (method, controls, sample size, fair test).
-Close with:
-- Still running / open — each live item: where it is and when it lands.
-- Corrections — anything that reversed an earlier belief (integrity first).
-
-## Scope
-Honor any window ("last 3 days") or sub-area named. Default: since the last report, or
-the currently-active work.
\ No newline at end of file
diff --git a/.claude/skills/slurm-harvest/SKILL.md b/.claude/skills/slurm-harvest/SKILL.md
deleted file mode 100644
index 5f54a743b..000000000
--- a/.claude/skills/slurm-harvest/SKILL.md
+++ /dev/null
@@ -1,38 +0,0 @@
----
-name: harvest-slurm
-description: Extract durable compute provenance (node, GPU/CPU specs, wall time, job ids, fitness/result lines) from SLURM stdout/stderr logs into research/results/compute_provenance.md, so the raw per-task logs can be deleted without losing reproducibility metadata. Use when a run's SLURM logs pile up and you want the proof-of-run kept in the research record, not as thousands of scheduler files.
-user-invocable: true
-disable-model-invocation: false
-argument-hint: [glob of slurm logs] [run-id]
----
-
-# Harvest SLURM logs → compute provenance
-
-SLURM job logs (`*.out`/`*.err`, `slurm-*.out`, array logs) are scheduler noise, not
-research records — but they carry real proof-of-run: node, GPU model + memory, gpu/cpu
-counts, start/wall time, job ids, and any fitness/result lines the job printed. This skill
-harvests those facts into `research/results/compute_provenance.md` (keyed by LOG run-id),
-after which the raw logs are safe to delete.
-
-## Procedure
-
-1. **Identify the log group** and the LOG run-id it belongs to (map job-name → run via
-   `research/LOG.md`). Group by job family if a run produced several (train + eval array, ES
-   coord + workers, etc.).
-2. **Harvest** (prints a Markdown block; add `--json out.json` for machine-readable):
-   ```
-   python .claude/skills/harvest-slurm/harvest_slurm.py --md \
-     --label "<run name>" --run-id <YYYY-MM-DD-rNN> --glob '<path/to/*.out>'
-   ```
-   It extracts: node(s), GPU model+MiB, gpu/cpu per job, start times, per-cell wall seconds
-   (summed), ES per-gen fitness trajectory, aggregate fitness units (n/mean/range), job ids,
-   and a count of files with error/traceback markers. Absent fields are simply omitted.
-3. **Append the block** to `research/results/compute_provenance.md` under the right run-id.
-   Note in it that the raw logs were harvested then deleted.
-4. **Delete the raw logs** once the provenance is captured. Do not add them to `.gitignore`
-   — the workflow no longer dumps logs into the repo; if any reappear, harvest + delete again.
-5. **Sanity-check** the harvested numbers against the run's own result JSON / LOG entry
-   (e.g. ES fitness trajectory should match the recorded verdict). Flag discrepancies.
-
-The extractor is pure-stdlib regex over heterogeneous formats; if a new log family prints
-specs differently, extend the patterns in `harvest_slurm.py`.
diff --git a/.claude/skills/slurm-harvest/harvest_slurm.py b/.claude/skills/slurm-harvest/harvest_slurm.py
deleted file mode 100644
index f07cb404c..000000000
--- a/.claude/skills/slurm-harvest/harvest_slurm.py
+++ /dev/null
@@ -1,190 +0,0 @@
-#!/usr/bin/env python3
-"""Harvest compute provenance from SLURM stdout/stderr logs, then they can be deleted.
-
-Scans a set of SLURM log files and extracts the durable "proof of run" facts —
-node(s), GPU model + memory, gpu/cpu counts, start time, wall time, job ids, and a
-sample of result/fitness lines — into a compact JSON + Markdown summary. The summary
-is meant to live in the research record (e.g. research/results/compute_provenance.md)
-so the raw per-task logs can be removed without losing reproducibility metadata.
-
-Pure stdlib. Robust to heterogeneous formats (training logs, ES coord/eval, eval
-arrays, probe units) via regex signal extraction — absent fields are simply omitted.
-
-Usage:
-    python harvest_slurm.py --label "ES run (E009)" --run-id 2026-06-03-r01 \
-        --glob 'scripts/stage_b/slurm_logs/*.out' [--json out.json] [--md]
-"""
-
-from __future__ import annotations
-
-import argparse
-import glob as globmod
-import json
-import re
-import sys
-from collections import Counter
-
-# --- signal patterns (lenient; match whatever a given log family happens to print)
-RE_GPU = re.compile(r"(NVIDIA[\w .\-]*?,\s*\d+\s*MiB|NVIDIA[\w .\-]+-\d+GB)")
-RE_NODE = re.compile(
-    r"\bnode:\s*([\w\-]+)|on\s+([\w\-]+)\s*=*$|coordinator:.*node:\s*([\w\-]+)"
-)
-RE_NODE2 = re.compile(r"\bnode:\s*([\w\-]+)")
-RE_GPUS = re.compile(r"\bgpus:\s*(\d+)")
-RE_CPUS = re.compile(r"\bcpus:\s*(\d+)")
-RE_STARTED = re.compile(r"\bstarted:\s*(.+)$")
-RE_JOB = re.compile(r"\bjob:\s*(\d+)")
-RE_CELL_SECS = re.compile(r"\((\d+)s\)")  # eval array: "(887s)"
-RE_FIT_COORD = re.compile(r"coord gen (\d+):\s*fit_mean=([-\d.]+)\s*fit_max=([-\d.]+)")
-RE_FITNESS = re.compile(r"FITNESS\s+mean_per_cog=([-\d.]+)")
-RE_ERROR = re.compile(
-    r"\b(Traceback|Error|CUDA out of memory|Killed|oom-kill|FAILED)\b", re.I
-)
-
-
-def harvest(paths: list[str]) -> dict:
-    nodes, gpus_models, jobs = Counter(), Counter(), set()
-    gpu_count = cpu_count = None
-    starts: list[str] = []
-    cell_secs: list[int] = []
-    coord_series: list[tuple[int, float, float]] = []
-    fitness_vals: list[float] = []
-    errors = 0
-    n = 0
-    for p in paths:
-        n += 1
-        try:
-            text = open(p, errors="replace").read()
-        except OSError:
-            continue
-        for m in RE_GPU.finditer(text):
-            gpus_models[m.group(1).strip()] += 1
-        for m in RE_NODE2.finditer(text):
-            nodes[m.group(1)] += 1
-        if g := RE_GPUS.search(text):
-            gpu_count = g.group(1)
-        if c := RE_CPUS.search(text):
-            cpu_count = c.group(1)
-        for m in RE_STARTED.finditer(text):
-            starts.append(m.group(1).strip())
-        for m in RE_JOB.finditer(text):
-            jobs.add(m.group(1))
-        cell_secs += [int(s) for s in RE_CELL_SECS.findall(text)]
-        for m in RE_FIT_COORD.finditer(text):
-            coord_series.append((int(m.group(1)), float(m.group(2)), float(m.group(3))))
-        fitness_vals += [float(v) for v in RE_FITNESS.findall(text)]
-        if RE_ERROR.search(text):
-            errors += 1
-
-    out: dict = {"n_files": n}
-    if nodes:
-        out["nodes"] = sorted(nodes)
-    if gpus_models:
-        out["gpu_models"] = sorted(gpus_models)
-    if gpu_count:
-        out["gpus_per_job"] = gpu_count
-    if cpu_count:
-        out["cpus_per_job"] = cpu_count
-    if starts:
-        out["started"] = sorted(set(starts))[:4]
-    if jobs:
-        out["job_ids_sample"] = sorted(jobs)[:8]
-        out["n_job_ids"] = len(jobs)
-    if cell_secs:
-        out["cell_wall_seconds"] = {
-            "n": len(cell_secs),
-            "sum_s": sum(cell_secs),
-            "sum_h": round(sum(cell_secs) / 3600, 2),
-            "max_s": max(cell_secs),
-        }
-    if coord_series:
-        coord_series.sort()
-        out["es_fitness_by_gen"] = [
-            {"gen": g, "fit_mean": fm, "fit_max": fx} for g, fm, fx in coord_series
-        ]
-    if fitness_vals:
-        out["fitness_units"] = {
-            "n": len(fitness_vals),
-            "mean": round(sum(fitness_vals) / len(fitness_vals), 4),
-            "min": min(fitness_vals),
-            "max": max(fitness_vals),
-        }
-    if errors:
-        out["files_with_error_markers"] = errors
-    return out
-
-
-def to_md(label: str, run_id: str, src: str, h: dict) -> str:
-    lines = [
-        f"### {label}" + (f"  (run {run_id})" if run_id else ""),
-        f"- Source logs: `{src}` — {h.get('n_files', 0)} files (harvested, then deleted).",
-    ]
-    if "gpu_models" in h:
-        lines.append(
-            f"- GPU: {', '.join(h['gpu_models'])}"
-            + (
-                f"; {h['gpus_per_job']} gpu / {h.get('cpus_per_job','?')} cpu per job"
-                if "gpus_per_job" in h
-                else ""
-            )
-        )
-    if "nodes" in h:
-        ns = h["nodes"]
-        lines.append(f"- Node(s): {', '.join(ns[:8])}" + (" …" if len(ns) > 8 else ""))
-    if "started" in h:
-        lines.append(f"- Started: {'; '.join(h['started'])}")
-    if "cell_wall_seconds" in h:
-        cw = h["cell_wall_seconds"]
-        lines.append(
-            f"- Eval wall: {cw['n']} cells, Σ {cw['sum_h']} h (max cell {cw['max_s']} s)."
-        )
-    if "fitness_units" in h:
-        fu = h["fitness_units"]
-        lines.append(
-            f"- Fitness units: n={fu['n']}, mean={fu['mean']}, range [{fu['min']}, {fu['max']}]."
-        )
-    if "es_fitness_by_gen" in h:
-        s = h["es_fitness_by_gen"]
-        traj = ", ".join(f"g{d['gen']}:{d['fit_mean']:.3f}" for d in s)
-        lines.append(f"- ES fitness/gen ({len(s)} gens): {traj}")
-    if "n_job_ids" in h:
-        lines.append(
-            f"- SLURM job ids: {h['n_job_ids']} (e.g. {', '.join(h['job_ids_sample'])})."
-        )
-    if "files_with_error_markers" in h:
-        lines.append(
-            f"- ⚠ {h['files_with_error_markers']} file(s) contained error/traceback markers."
-        )
-    return "\n".join(lines)
-
-
-def main(argv=None) -> int:
-    ap = argparse.ArgumentParser(description=__doc__)
-    ap.add_argument("--glob", required=True, help="glob for the log files")
-    ap.add_argument("--label", required=True)
-    ap.add_argument("--run-id", default="")
-    ap.add_argument("--json", type=str, default="")
-    ap.add_argument("--md", action="store_true", help="print a Markdown block")
-    args = ap.parse_args(argv)
-
-    paths = sorted(globmod.glob(args.glob))
-    if not paths:
-        print(f"no files matched: {args.glob}", file=sys.stderr)
-        return 1
-    h = harvest(paths)
-    if args.json:
-        json.dump(
-            {"label": args.label, "run_id": args.run_id, "glob": args.glob, **h},
-            open(args.json, "w"),
-            indent=2,
-        )
-    print(
-        to_md(args.label, args.run_id, args.glob, h)
-        if args.md
-        else json.dumps(h, indent=2)
-    )
-    return 0
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())
diff --git a/.gitignore b/.gitignore
index 71aadef8d..c11d4ba62 100644
--- a/.gitignore
+++ b/.gitignore
@@ -23,4 +23,6 @@ dist/*
 logs/*
 .pytest_cache/*
 .vscode/*
-data/*
\ No newline at end of file
+data/*
+.claude/*
+.claude