Implement custom add_value methods with add_value_fn argument in EventStabilityTracker.

detectmatelibrary_tests/test_detectors/test_bigram_frequency_detector.py

@@ -255,7 +255,7 @@ def test_audit_log_anomalies(self):
             if detector.detect(log, output_=output):
                 detected_ids.add(log["logID"])
 
-        assert detected_ids == {'1859', '1860', '1861', '1862', '1864', '1865', '1866', '1867'}
+        assert detected_ids == {'1859', '1860', '1861', '1862'}
 
 
 class TestBigramFrequencyDetectorAutoConfig:
@@ -288,7 +288,7 @@ def test_audit_log_anomalies_via_process(self):
             if detector.process(log) is not None:
                 detected_ids.add(log["logID"])
 
-        assert detected_ids == {'1859', '1860', '1861', '1862', '1864', '1865', '1866', '1867'}
+        assert detected_ids == {'1859', '1860', '1861', '1862'}
 
 
 class TestBigramFrequencyDetectorGlobalInstances:

detectmatelibrary_tests/test_detectors/test_charset_detector.py

@@ -83,7 +83,7 @@ def test_custom_config_initialization(self):
         assert hasattr(detector, 'persistency')
         assert isinstance(detector.persistency.events_data, dict)
 
-    def test_persistency_uses_expand_value(self):
+    def test_persistency_uses_custom_add_value(self):
         """Main persistency must accumulate characters; auto_conf must not."""
         detector = CharsetDetector()
         # Ingest a sample so a SingleStabilityTracker is materialized
@@ -93,20 +93,8 @@ def test_persistency_uses_expand_value(self):
             named_variables={"v": "hello"},
         )
         single = detector.persistency.get_event_data(1)["v"]
-        assert single.expand_value is True
         assert single.unique_set == {"h", "e", "l", "o"}
 
-    def test_auto_conf_persistency_does_not_expand(self):
-        detector = CharsetDetector()
-        detector.auto_conf_persistency.ingest_event(
-            event_id=1,
-            event_template="t",
-            named_variables={"v": "hello"},
-        )
-        single = detector.auto_conf_persistency.get_event_data(1)["v"]
-        assert single.expand_value is False
-        assert single.unique_set == {"hello"}
-
     def test_register_persistency_was_called(self):
         """Main persistency should be registered so persist/load round-trips
         work."""

detectmatelibrary_tests/test_persistency/test_stability_tracker.py

@@ -5,32 +5,32 @@
 
 
 class TestSingleStabilityTrackerExpandValue:
-    def test_default_add_stores_whole_value(self):
+    def test_default_add_value(self):
         tracker = SingleStabilityTracker()
         tracker.add_value("hello")
         tracker.add_value("world")
         assert tracker.unique_set == {"hello", "world"}
 
-    def test_expand_value_unions_characters(self):
-        tracker = SingleStabilityTracker(expand_value=True)
+    def test_custom_add_value(self):
+        tracker = SingleStabilityTracker(add_value_fn="CharsetDetector")
         tracker.add_value("hello")
         tracker.add_value("world")
         assert tracker.unique_set == {"h", "e", "l", "o", "w", "r", "d"}
 
     def test_expand_value_change_series_tracks_growth(self):
-        tracker = SingleStabilityTracker(expand_value=True)
+        tracker = SingleStabilityTracker(add_value_fn="CharsetDetector")
         tracker.add_value("ab")           # adds {a, b}, change=True
         tracker.add_value("ba")           # adds nothing new, change=False
         tracker.add_value("c")            # adds {c}, change=True
         assert list(tracker.change_series) == [True, False, True]
 
     def test_expand_value_round_trip(self):
-        tracker = SingleStabilityTracker(expand_value=True)
+        tracker = SingleStabilityTracker(add_value_fn="CharsetDetector")
         tracker.add_value("hello")
         tracker.add_value("world")
         state = tracker.to_state()
         restored = SingleStabilityTracker.from_state(state)
-        assert restored.expand_value is True
+        assert restored.add_value_fn == "CharsetDetector"
         assert restored.unique_set == {"h", "e", "l", "o", "w", "r", "d"}
         # subsequent ingestion still unions characters
         restored.add_value("xy")
@@ -40,9 +40,8 @@ def test_legacy_state_without_expand_value_defaults_false(self):
         tracker = SingleStabilityTracker()
         tracker.add_value("hello")
         state = tracker.to_state()
-        state.pop("expand_value", None)  # simulate pre-flag snapshot
         restored = SingleStabilityTracker.from_state(state)
-        assert restored.expand_value is False
+        assert restored.add_value_fn == "default"
         assert restored.unique_set == {"hello"}
 
 
@@ -51,45 +50,51 @@ def test_default_event_tracker_uses_add_semantics(self):
         event_tracker = EventStabilityTracker()
         event_tracker.add_data({"var1": "hello"})
         single = event_tracker.get_data()["var1"]
-        assert single.expand_value is False
+        assert single.add_value_fn == "default"
+        assert single.detector_config is None
         assert single.unique_set == {"hello"}
 
     def test_expand_value_propagates_to_per_variable_trackers(self):
-        event_tracker = EventStabilityTracker(expand_value=True)
+        event_tracker = EventStabilityTracker(add_value_fn="CharsetDetector")
         event_tracker.add_data({"var1": "hello"})
         event_tracker.add_data({"var1": "world"})
         single = event_tracker.get_data()["var1"]
-        assert single.expand_value is True
+        assert single.add_value_fn == "CharsetDetector"
+        assert single.detector_config is None
         assert single.unique_set == {"h", "e", "l", "o", "w", "r", "d"}
 
     def test_each_new_variable_gets_its_own_configured_tracker(self):
-        event_tracker = EventStabilityTracker(expand_value=True)
+        event_tracker = EventStabilityTracker(add_value_fn="CharsetDetector")
         event_tracker.add_data({"a": "ab", "b": "cd"})
         a = event_tracker.get_data()["a"]
         b = event_tracker.get_data()["b"]
-        assert a.expand_value is True
-        assert b.expand_value is True
+        assert a.add_value_fn == "CharsetDetector"
+        assert b.add_value_fn == "CharsetDetector"
+        assert a.detector_config is None
+        assert b.detector_config is None
         assert a.unique_set == {"a", "b"}
         assert b.unique_set == {"c", "d"}
 
     def test_post_load_new_variable_honors_expand_value(self):
         """After dump/load, a variable that wasn't present at save time should
         still use expand_value semantics when first ingested."""
-        original = EventStabilityTracker(expand_value=True)
+        original = EventStabilityTracker(add_value_fn="CharsetDetector")
         original.add_data({"known": "abc"})
         blob = original.dump()
-        restored = EventStabilityTracker.load(blob, expand_value=True)
+        restored = EventStabilityTracker.load(blob, add_value_fn="CharsetDetector")
 
         # Ingest a brand-new variable not present in the saved state
         restored.add_data({"known": "de", "newvar": "xy"})
 
         new_tracker = restored.get_data()["newvar"]
-        assert new_tracker.expand_value is True
+        assert new_tracker.add_value_fn == "CharsetDetector"
+        assert new_tracker.detector_config is None
         assert new_tracker.unique_set == {"x", "y"}
 
         # And the existing variable continues to expand correctly
         known_tracker = restored.get_data()["known"]
-        assert known_tracker.expand_value is True
+        assert known_tracker.add_value_fn == "CharsetDetector"
+        assert known_tracker.detector_config is None
         assert {"a", "b", "c", "d", "e"} <= known_tracker.unique_set
 
 

src/detectmatelibrary/detectors/__init__.py

@@ -1,3 +1,4 @@
+from .bigram_frequency_detector import BigramFrequencyDetector, BigramFrequencyDetectorConfig
 from .random_detector import RandomDetector, RandomDetectorConfig
 from .new_value_detector import NewValueDetector, NewValueDetectorConfig
 from .new_event_detector import NewEventDetector, NewEventDetectorConfig
@@ -15,5 +16,7 @@
     "ValueRangeDetector",
     "ValueRangeDetectorConfig",
     "CharsetDetector",
-    "CharsetDetectorConfig"
+    "CharsetDetectorConfig",
+    "BigramFrequencyDetector",
+    "BigramFrequencyDetectorConfig"
 ]

src/detectmatelibrary/detectors/bigram_frequency_detector.py

@@ -1,5 +1,4 @@
 from typing import Any, cast
-
 from detectmatelibrary.common._config._compile import generate_detector_config
 from detectmatelibrary.common._config._formats import EventsConfig
 from detectmatelibrary.common.detector import (
@@ -67,18 +66,57 @@ def __init__(
         name: str = "BigramFrequencyDetector",
         config: BigramFrequencyDetectorConfig = BigramFrequencyDetectorConfig()
     ) -> None:
-
         if isinstance(config, dict):
             config = BigramFrequencyDetectorConfig.from_dict(config, name)
 
+        def add_value(cls: SingleStabilityTracker, value: Any) -> None:
+            """Add a new value to the tracker."""
+            change = False
+            default_freq, default_total = (_default_freq_tables() if self.config.default_freqs else ({}, {}))
+            freq: dict[Any, dict[Any, int]] = cls.extra_state.get("freq", {})
+            total_freq: dict[Any, int] = cls.extra_state.get("total_freq", {})
+            probs: list[float] = []
+            for i in range(-1, len(value)):
+                first: Any = -1 if i == -1 else value[i]
+                second: Any = -1 if i == len(value) - 1 else value[i + 1]
+                prob = 0.0
+                if first in freq and second in freq[first] and total_freq.get(first, 0) > 0:
+                    prob = freq[first][second] / total_freq[first]
+                elif self.config.default_freqs:
+                    if (first in default_freq and second in default_freq[first]
+                            and default_total.get(first, 0) > 0):
+                        prob = default_freq[first][second] / default_total[first]
+                probs.append(prob)
+            if probs:
+                critical_val = sum(probs) / len(probs)
+                change = critical_val > self.config.prob_thresh or any(x == 0.0 for x in probs)
+
+            if self.config.skip_repetitions and value in cls.unique_set:
+                change = False
+            else:
+                for i in range(-1, len(value)):
+                    first = -1 if i == -1 else value[i]
+                    second = -1 if i == len(value) - 1 else value[i + 1]
+                    row = freq.setdefault(first, {})
+                    row[second] = row.get(second, 0) + 1
+                    total_freq[first] = total_freq.get(first, 0) + 1
+            cls.unique_set.add(value)
+            cls.change_series.append(change)
+        self.add_value_fn = add_value
+
         super().__init__(name=name, buffer_mode=BufferMode.NO_BUF, config=config)
         self.config: BigramFrequencyDetectorConfig  # type narrowing for IDE
+        kwargs = {"add_value_fn": self.__class__.__name__, "detector_config": self.config.to_dict(
+            method_id="BigramFrequencyDetector")}
         self.persistency = EventPersistency(
             event_data_class=EventStabilityTracker,
+            event_data_kwargs=kwargs
+
         )
-        # auto config checks if individual variables are stable to select combos from
+        # auto config checks if individual variables are stable
         self.auto_conf_persistency = EventPersistency(
-            event_data_class=EventStabilityTracker
+            event_data_class=EventStabilityTracker,
+            event_data_kwargs=kwargs
         )
         self._register_persistency(self.persistency)
 
@@ -99,17 +137,13 @@ def train(self, input_: ParserSchema) -> None:  # type: ignore
             named_variables=configured_variables,
         )
         if configured_variables:
-            known_events = cast(
-                dict[int | str, EventStabilityTracker], self.persistency.get_events_data()
-            )
+            known_events = cast(dict[int | str, EventStabilityTracker], self.persistency.get_events_data())
             self.train_helper(configured_variables, current_event_id, known_events, pre_unique)
 
         if self.config.global_instances:
             global_vars = get_global_variables(input_, self.config.global_instances)
             if global_vars:
-                pre_unique_global = self._snapshot_unique_sets(
-                    known_events.get(GLOBAL_EVENT_ID), global_vars
-                )
+                pre_unique_global = self._snapshot_unique_sets(known_events.get(GLOBAL_EVENT_ID), global_vars)
                 self.persistency.ingest_event(
                     event_id=GLOBAL_EVENT_ID,
                     event_template=input_["template"],
@@ -175,9 +209,7 @@ def detect(
         configured_variables = get_configured_variables(input_, self.config.events)
         overall_score = 0.0
         current_event_id = input_["EventID"]
-        known_events = cast(
-            dict[int | str, EventStabilityTracker], self.persistency.get_events_data()
-        )
+        known_events = cast(dict[int | str, EventStabilityTracker], self.persistency.get_events_data())
         if current_event_id in known_events:
             overall_score = self.detect_helper(
                 alerts, configured_variables, current_event_id, known_events, overall_score
@@ -203,12 +235,8 @@ def detect_helper(
         overall_score: float,
     ) -> float:
         anomaly = False
-        default_freq, default_total = (
-            _default_freq_tables() if self.config.default_freqs else ({}, {})
-        )
-        var_trackers = cast(
-            dict[str, SingleStabilityTracker], known_events[event_id].get_data()
-        )
+        default_freq, default_total = (_default_freq_tables() if self.config.default_freqs else ({}, {}))
+        var_trackers = cast(dict[str, SingleStabilityTracker], known_events[event_id].get_data())
         for var_name, single_tracker in var_trackers.items():
             value: Any = variables.get(var_name)
             if value is None:

src/detectmatelibrary/detectors/charset_detector.py

@@ -1,3 +1,4 @@
+from typing import Any
 from detectmatelibrary.common._config._compile import generate_detector_config
 from detectmatelibrary.common._config._formats import EventsConfig
 from detectmatelibrary.common.detector import (
@@ -8,7 +9,8 @@
     validate_config_coverage,
 )
 from detectmatelibrary.utils.persistency.event_data_structures.trackers.stability.stability_tracker import (
-    EventStabilityTracker
+    EventStabilityTracker,
+    SingleStabilityTracker
 )
 from detectmatelibrary.utils.persistency.event_persistency import EventPersistency
 from detectmatelibrary.utils.data_buffer import BufferMode
@@ -37,14 +39,26 @@ def __init__(
         if isinstance(config, dict):
             config = CharsetDetectorConfig.from_dict(config, name)
 
+        def add_value(cls: SingleStabilityTracker, value: Any) -> None:
+            """Add a new value to the tracker."""
+            before = len(cls.unique_set)
+            cls.unique_set.update(value)
+            cls.change_series.append(len(cls.unique_set) > before)
+        self.add_value_fn = add_value
+
         super().__init__(name=name, buffer_mode=BufferMode.NO_BUF, config=config)
         self.config: CharsetDetectorConfig  # type narrowing for IDE
+        kwargs = {"add_value_fn": self.__class__.__name__, "detector_config": self.config.to_dict(
+            method_id="CharsetDetector")}
         self.persistency = EventPersistency(
             event_data_class=EventStabilityTracker,
-            event_data_kwargs={"expand_value": True},
+            event_data_kwargs=kwargs
+        )
+        # auto config checks if individual variables are stable to select characters from
+        self.auto_conf_persistency = EventPersistency(
+            event_data_class=EventStabilityTracker,
+            event_data_kwargs=kwargs
         )
-        # auto config checks if individual variables are stable to select combos from
-        self.auto_conf_persistency = EventPersistency(event_data_class=EventStabilityTracker)
         self._register_persistency(self.persistency)
 
     def train(self, input_: ParserSchema) -> None:  # type: ignore