diff --git a/detectmatelibrary_tests/test_detectors/test_bigram_frequency_detector.py b/detectmatelibrary_tests/test_detectors/test_bigram_frequency_detector.py index 3e678d47..22937cee 100644 --- a/detectmatelibrary_tests/test_detectors/test_bigram_frequency_detector.py +++ b/detectmatelibrary_tests/test_detectors/test_bigram_frequency_detector.py @@ -255,7 +255,7 @@ def test_audit_log_anomalies(self): if detector.detect(log, output_=output): detected_ids.add(log["logID"]) - assert detected_ids == {'1859', '1860', '1861', '1862', '1864', '1865', '1866', '1867'} + assert detected_ids == {'1859', '1860', '1861', '1862'} class TestBigramFrequencyDetectorAutoConfig: @@ -288,7 +288,7 @@ def test_audit_log_anomalies_via_process(self): if detector.process(log) is not None: detected_ids.add(log["logID"]) - assert detected_ids == {'1859', '1860', '1861', '1862', '1864', '1865', '1866', '1867'} + assert detected_ids == {'1859', '1860', '1861', '1862'} class TestBigramFrequencyDetectorGlobalInstances: diff --git a/detectmatelibrary_tests/test_detectors/test_charset_detector.py b/detectmatelibrary_tests/test_detectors/test_charset_detector.py index d88fdfdd..8fe52531 100644 --- a/detectmatelibrary_tests/test_detectors/test_charset_detector.py +++ b/detectmatelibrary_tests/test_detectors/test_charset_detector.py @@ -83,7 +83,7 @@ def test_custom_config_initialization(self): assert hasattr(detector, 'persistency') assert isinstance(detector.persistency.events_data, dict) - def test_persistency_uses_expand_value(self): + def test_persistency_uses_custom_add_value(self): """Main persistency must accumulate characters; auto_conf must not.""" detector = CharsetDetector() # Ingest a sample so a SingleStabilityTracker is materialized @@ -93,20 +93,8 @@ def test_persistency_uses_expand_value(self): named_variables={"v": "hello"}, ) single = detector.persistency.get_event_data(1)["v"] - assert single.expand_value is True assert single.unique_set == {"h", "e", "l", "o"} - def test_auto_conf_persistency_does_not_expand(self): - detector = CharsetDetector() - detector.auto_conf_persistency.ingest_event( - event_id=1, - event_template="t", - named_variables={"v": "hello"}, - ) - single = detector.auto_conf_persistency.get_event_data(1)["v"] - assert single.expand_value is False - assert single.unique_set == {"hello"} - def test_register_persistency_was_called(self): """Main persistency should be registered so persist/load round-trips work.""" diff --git a/detectmatelibrary_tests/test_persistency/test_stability_tracker.py b/detectmatelibrary_tests/test_persistency/test_stability_tracker.py index d6aa34a6..894c9888 100644 --- a/detectmatelibrary_tests/test_persistency/test_stability_tracker.py +++ b/detectmatelibrary_tests/test_persistency/test_stability_tracker.py @@ -5,32 +5,32 @@ class TestSingleStabilityTrackerExpandValue: - def test_default_add_stores_whole_value(self): + def test_default_add_value(self): tracker = SingleStabilityTracker() tracker.add_value("hello") tracker.add_value("world") assert tracker.unique_set == {"hello", "world"} - def test_expand_value_unions_characters(self): - tracker = SingleStabilityTracker(expand_value=True) + def test_custom_add_value(self): + tracker = SingleStabilityTracker(add_value_fn="CharsetDetector") tracker.add_value("hello") tracker.add_value("world") assert tracker.unique_set == {"h", "e", "l", "o", "w", "r", "d"} def test_expand_value_change_series_tracks_growth(self): - tracker = SingleStabilityTracker(expand_value=True) + tracker = SingleStabilityTracker(add_value_fn="CharsetDetector") tracker.add_value("ab") # adds {a, b}, change=True tracker.add_value("ba") # adds nothing new, change=False tracker.add_value("c") # adds {c}, change=True assert list(tracker.change_series) == [True, False, True] def test_expand_value_round_trip(self): - tracker = SingleStabilityTracker(expand_value=True) + tracker = SingleStabilityTracker(add_value_fn="CharsetDetector") tracker.add_value("hello") tracker.add_value("world") state = tracker.to_state() restored = SingleStabilityTracker.from_state(state) - assert restored.expand_value is True + assert restored.add_value_fn == "CharsetDetector" assert restored.unique_set == {"h", "e", "l", "o", "w", "r", "d"} # subsequent ingestion still unions characters restored.add_value("xy") @@ -40,9 +40,8 @@ def test_legacy_state_without_expand_value_defaults_false(self): tracker = SingleStabilityTracker() tracker.add_value("hello") state = tracker.to_state() - state.pop("expand_value", None) # simulate pre-flag snapshot restored = SingleStabilityTracker.from_state(state) - assert restored.expand_value is False + assert restored.add_value_fn == "default" assert restored.unique_set == {"hello"} @@ -51,45 +50,51 @@ def test_default_event_tracker_uses_add_semantics(self): event_tracker = EventStabilityTracker() event_tracker.add_data({"var1": "hello"}) single = event_tracker.get_data()["var1"] - assert single.expand_value is False + assert single.add_value_fn == "default" + assert single.detector_config is None assert single.unique_set == {"hello"} def test_expand_value_propagates_to_per_variable_trackers(self): - event_tracker = EventStabilityTracker(expand_value=True) + event_tracker = EventStabilityTracker(add_value_fn="CharsetDetector") event_tracker.add_data({"var1": "hello"}) event_tracker.add_data({"var1": "world"}) single = event_tracker.get_data()["var1"] - assert single.expand_value is True + assert single.add_value_fn == "CharsetDetector" + assert single.detector_config is None assert single.unique_set == {"h", "e", "l", "o", "w", "r", "d"} def test_each_new_variable_gets_its_own_configured_tracker(self): - event_tracker = EventStabilityTracker(expand_value=True) + event_tracker = EventStabilityTracker(add_value_fn="CharsetDetector") event_tracker.add_data({"a": "ab", "b": "cd"}) a = event_tracker.get_data()["a"] b = event_tracker.get_data()["b"] - assert a.expand_value is True - assert b.expand_value is True + assert a.add_value_fn == "CharsetDetector" + assert b.add_value_fn == "CharsetDetector" + assert a.detector_config is None + assert b.detector_config is None assert a.unique_set == {"a", "b"} assert b.unique_set == {"c", "d"} def test_post_load_new_variable_honors_expand_value(self): """After dump/load, a variable that wasn't present at save time should still use expand_value semantics when first ingested.""" - original = EventStabilityTracker(expand_value=True) + original = EventStabilityTracker(add_value_fn="CharsetDetector") original.add_data({"known": "abc"}) blob = original.dump() - restored = EventStabilityTracker.load(blob, expand_value=True) + restored = EventStabilityTracker.load(blob, add_value_fn="CharsetDetector") # Ingest a brand-new variable not present in the saved state restored.add_data({"known": "de", "newvar": "xy"}) new_tracker = restored.get_data()["newvar"] - assert new_tracker.expand_value is True + assert new_tracker.add_value_fn == "CharsetDetector" + assert new_tracker.detector_config is None assert new_tracker.unique_set == {"x", "y"} # And the existing variable continues to expand correctly known_tracker = restored.get_data()["known"] - assert known_tracker.expand_value is True + assert known_tracker.add_value_fn == "CharsetDetector" + assert known_tracker.detector_config is None assert {"a", "b", "c", "d", "e"} <= known_tracker.unique_set diff --git a/src/detectmatelibrary/detectors/__init__.py b/src/detectmatelibrary/detectors/__init__.py index 361db003..9fa88739 100644 --- a/src/detectmatelibrary/detectors/__init__.py +++ b/src/detectmatelibrary/detectors/__init__.py @@ -1,3 +1,4 @@ +from .bigram_frequency_detector import BigramFrequencyDetector, BigramFrequencyDetectorConfig from .random_detector import RandomDetector, RandomDetectorConfig from .new_value_detector import NewValueDetector, NewValueDetectorConfig from .new_event_detector import NewEventDetector, NewEventDetectorConfig @@ -15,5 +16,7 @@ "ValueRangeDetector", "ValueRangeDetectorConfig", "CharsetDetector", - "CharsetDetectorConfig" + "CharsetDetectorConfig", + "BigramFrequencyDetector", + "BigramFrequencyDetectorConfig" ] diff --git a/src/detectmatelibrary/detectors/bigram_frequency_detector.py b/src/detectmatelibrary/detectors/bigram_frequency_detector.py index 4daf163a..934bf880 100644 --- a/src/detectmatelibrary/detectors/bigram_frequency_detector.py +++ b/src/detectmatelibrary/detectors/bigram_frequency_detector.py @@ -1,5 +1,4 @@ from typing import Any, cast - from detectmatelibrary.common._config._compile import generate_detector_config from detectmatelibrary.common._config._formats import EventsConfig from detectmatelibrary.common.detector import ( @@ -67,18 +66,57 @@ def __init__( name: str = "BigramFrequencyDetector", config: BigramFrequencyDetectorConfig = BigramFrequencyDetectorConfig() ) -> None: - if isinstance(config, dict): config = BigramFrequencyDetectorConfig.from_dict(config, name) + def add_value(cls: SingleStabilityTracker, value: Any) -> None: + """Add a new value to the tracker.""" + change = False + default_freq, default_total = (_default_freq_tables() if self.config.default_freqs else ({}, {})) + freq: dict[Any, dict[Any, int]] = cls.extra_state.get("freq", {}) + total_freq: dict[Any, int] = cls.extra_state.get("total_freq", {}) + probs: list[float] = [] + for i in range(-1, len(value)): + first: Any = -1 if i == -1 else value[i] + second: Any = -1 if i == len(value) - 1 else value[i + 1] + prob = 0.0 + if first in freq and second in freq[first] and total_freq.get(first, 0) > 0: + prob = freq[first][second] / total_freq[first] + elif self.config.default_freqs: + if (first in default_freq and second in default_freq[first] + and default_total.get(first, 0) > 0): + prob = default_freq[first][second] / default_total[first] + probs.append(prob) + if probs: + critical_val = sum(probs) / len(probs) + change = critical_val > self.config.prob_thresh or any(x == 0.0 for x in probs) + + if self.config.skip_repetitions and value in cls.unique_set: + change = False + else: + for i in range(-1, len(value)): + first = -1 if i == -1 else value[i] + second = -1 if i == len(value) - 1 else value[i + 1] + row = freq.setdefault(first, {}) + row[second] = row.get(second, 0) + 1 + total_freq[first] = total_freq.get(first, 0) + 1 + cls.unique_set.add(value) + cls.change_series.append(change) + self.add_value_fn = add_value + super().__init__(name=name, buffer_mode=BufferMode.NO_BUF, config=config) self.config: BigramFrequencyDetectorConfig # type narrowing for IDE + kwargs = {"add_value_fn": self.__class__.__name__, "detector_config": self.config.to_dict( + method_id="BigramFrequencyDetector")} self.persistency = EventPersistency( event_data_class=EventStabilityTracker, + event_data_kwargs=kwargs + ) - # auto config checks if individual variables are stable to select combos from + # auto config checks if individual variables are stable self.auto_conf_persistency = EventPersistency( - event_data_class=EventStabilityTracker + event_data_class=EventStabilityTracker, + event_data_kwargs=kwargs ) self._register_persistency(self.persistency) @@ -99,17 +137,13 @@ def train(self, input_: ParserSchema) -> None: # type: ignore named_variables=configured_variables, ) if configured_variables: - known_events = cast( - dict[int | str, EventStabilityTracker], self.persistency.get_events_data() - ) + known_events = cast(dict[int | str, EventStabilityTracker], self.persistency.get_events_data()) self.train_helper(configured_variables, current_event_id, known_events, pre_unique) if self.config.global_instances: global_vars = get_global_variables(input_, self.config.global_instances) if global_vars: - pre_unique_global = self._snapshot_unique_sets( - known_events.get(GLOBAL_EVENT_ID), global_vars - ) + pre_unique_global = self._snapshot_unique_sets(known_events.get(GLOBAL_EVENT_ID), global_vars) self.persistency.ingest_event( event_id=GLOBAL_EVENT_ID, event_template=input_["template"], @@ -175,9 +209,7 @@ def detect( configured_variables = get_configured_variables(input_, self.config.events) overall_score = 0.0 current_event_id = input_["EventID"] - known_events = cast( - dict[int | str, EventStabilityTracker], self.persistency.get_events_data() - ) + known_events = cast(dict[int | str, EventStabilityTracker], self.persistency.get_events_data()) if current_event_id in known_events: overall_score = self.detect_helper( alerts, configured_variables, current_event_id, known_events, overall_score @@ -203,12 +235,8 @@ def detect_helper( overall_score: float, ) -> float: anomaly = False - default_freq, default_total = ( - _default_freq_tables() if self.config.default_freqs else ({}, {}) - ) - var_trackers = cast( - dict[str, SingleStabilityTracker], known_events[event_id].get_data() - ) + default_freq, default_total = (_default_freq_tables() if self.config.default_freqs else ({}, {})) + var_trackers = cast(dict[str, SingleStabilityTracker], known_events[event_id].get_data()) for var_name, single_tracker in var_trackers.items(): value: Any = variables.get(var_name) if value is None: diff --git a/src/detectmatelibrary/detectors/charset_detector.py b/src/detectmatelibrary/detectors/charset_detector.py index d3b48f5d..13edea1c 100644 --- a/src/detectmatelibrary/detectors/charset_detector.py +++ b/src/detectmatelibrary/detectors/charset_detector.py @@ -1,3 +1,4 @@ +from typing import Any from detectmatelibrary.common._config._compile import generate_detector_config from detectmatelibrary.common._config._formats import EventsConfig from detectmatelibrary.common.detector import ( @@ -8,7 +9,8 @@ validate_config_coverage, ) from detectmatelibrary.utils.persistency.event_data_structures.trackers.stability.stability_tracker import ( - EventStabilityTracker + EventStabilityTracker, + SingleStabilityTracker ) from detectmatelibrary.utils.persistency.event_persistency import EventPersistency from detectmatelibrary.utils.data_buffer import BufferMode @@ -37,14 +39,26 @@ def __init__( if isinstance(config, dict): config = CharsetDetectorConfig.from_dict(config, name) + def add_value(cls: SingleStabilityTracker, value: Any) -> None: + """Add a new value to the tracker.""" + before = len(cls.unique_set) + cls.unique_set.update(value) + cls.change_series.append(len(cls.unique_set) > before) + self.add_value_fn = add_value + super().__init__(name=name, buffer_mode=BufferMode.NO_BUF, config=config) self.config: CharsetDetectorConfig # type narrowing for IDE + kwargs = {"add_value_fn": self.__class__.__name__, "detector_config": self.config.to_dict( + method_id="CharsetDetector")} self.persistency = EventPersistency( event_data_class=EventStabilityTracker, - event_data_kwargs={"expand_value": True}, + event_data_kwargs=kwargs + ) + # auto config checks if individual variables are stable to select characters from + self.auto_conf_persistency = EventPersistency( + event_data_class=EventStabilityTracker, + event_data_kwargs=kwargs ) - # auto config checks if individual variables are stable to select combos from - self.auto_conf_persistency = EventPersistency(event_data_class=EventStabilityTracker) self._register_persistency(self.persistency) def train(self, input_: ParserSchema) -> None: # type: ignore diff --git a/src/detectmatelibrary/detectors/value_range_detector.py b/src/detectmatelibrary/detectors/value_range_detector.py index bc3a0815..67e54159 100644 --- a/src/detectmatelibrary/detectors/value_range_detector.py +++ b/src/detectmatelibrary/detectors/value_range_detector.py @@ -8,7 +8,8 @@ validate_config_coverage, ) from detectmatelibrary.utils.persistency.event_data_structures.trackers.stability.stability_tracker import ( - EventStabilityTracker + EventStabilityTracker, + SingleStabilityTracker ) from detectmatelibrary.utils.persistency.event_persistency import EventPersistency from detectmatelibrary.utils.data_buffer import BufferMode @@ -39,14 +40,34 @@ def __init__( if isinstance(config, dict): config = ValueRangeDetectorConfig.from_dict(config, name) + def add_value(cls: SingleStabilityTracker, value: int | float) -> None: + """Add a new value to the tracker.""" + try: + value = float(value) + value = int(value) if value.is_integer() else value + except ValueError: + return + if len(cls.unique_set) > 0: + min_ = min(cls.unique_set) + max_ = max(cls.unique_set) + cls.change_series.append(value < min_ or value > max_) + else: + cls.change_series.append(True) + cls.unique_set.add(value) + self.add_value_fn = add_value + super().__init__(name=name, buffer_mode=BufferMode.NO_BUF, config=config) self.config: ValueRangeDetectorConfig # type narrowing for IDE + kwargs = {"add_value_fn": self.__class__.__name__, "detector_config": self.config.to_dict( + method_id="ValueRangeDetector")} self.persistency = EventPersistency( event_data_class=EventStabilityTracker, + event_data_kwargs=kwargs ) - # auto config checks if individual variables are stable to select combos from + # auto config checks if individual variables are stable to select value ranges from self.auto_conf_persistency = EventPersistency( - event_data_class=EventStabilityTracker + event_data_class=EventStabilityTracker, + event_data_kwargs=kwargs ) def cast_val_to_numeric(self, configured_variables: Dict[str, Any], k: str, remove: List[str], @@ -54,17 +75,16 @@ def cast_val_to_numeric(self, configured_variables: Dict[str, Any], k: str, remo v = configured_variables[k] if not isinstance(v, (int, float)): try: - configured_variables[k] = int(v) + configured_variables[k] = float(v) + configured_variables[k] = int(configured_variables[k])\ + if configured_variables[k].is_integer() else configured_variables[k] except ValueError: - try: - configured_variables[k] = float(v) - except ValueError: - logger.error(f"Non-numeric value '{v}' appeared in {stage} of {self.__class__.__name__}" - f" with the name {self.name}.") - if not self.config.ignore_non_numerical_val: - exit(1) - remove.append(k) - return False + logger.error(f"Non-numeric value '{v}' appeared in {stage} of {self.__class__.__name__}" + f" with the name {self.name}.") + if not self.config.ignore_non_numerical_val: + exit(1) + remove.append(k) + return False return True def train(self, input_: ParserSchema) -> None: # type: ignore diff --git a/src/detectmatelibrary/utils/persistency/event_data_structures/trackers/stability/stability_tracker.py b/src/detectmatelibrary/utils/persistency/event_data_structures/trackers/stability/stability_tracker.py index 8184cc0a..c32ff8ac 100644 --- a/src/detectmatelibrary/utils/persistency/event_data_structures/trackers/stability/stability_tracker.py +++ b/src/detectmatelibrary/utils/persistency/event_data_structures/trackers/stability/stability_tracker.py @@ -1,34 +1,44 @@ """Tracks whether a variable is converging to a constant value.""" -from typing import Any, Callable, Dict, List, Literal, Set - +import importlib +from typing import Any, Callable, Dict, List, Literal, Set, TYPE_CHECKING from detectmatelibrary.utils.preview_helpers import list_preview_str from detectmatelibrary.utils.persistency.rle_list import RLEList - from ..base import SingleTracker, MultiTracker, EventTracker, Classification from .stability_classifier import StabilityClassifier +if TYPE_CHECKING: + from detectmatelibrary.common.detector import CoreDetectorConfig + class SingleStabilityTracker(SingleTracker): """Tracks stability of a single feature.""" - def __init__(self, min_samples: int = 3, expand_value: bool = False) -> None: + def __init__(self, min_samples: int = 3, add_value_fn: str = "default", + detector_config: "CoreDetectorConfig | None" = None) -> None: self.min_samples = min_samples - self.expand_value = expand_value self.change_series: RLEList[bool] = RLEList() self.unique_set: Set[Any] = set() self.stability_classifier: StabilityClassifier = StabilityClassifier( segment_thresholds=[1.1, 0.3, 0.1, 0.01], ) - self._accum = set.update if expand_value else set.add # Opaque slot for detectors to stash per-variable model state that # must survive save/load. Schema-free; the tracker does not interpret it. self.extra_state: Dict[str, Any] = {} + self.add_value_fn = add_value_fn + self.detector_config = detector_config + if add_value_fn != "default": + detector = getattr(importlib.import_module("detectmatelibrary.detectors"), add_value_fn) + if detector_config is not None: + detector = detector(config=detector_config) + else: + detector = detector() + self.add_value = detector.add_value_fn.__get__(self, type(self)) # type: ignore[method-assign] def add_value(self, value: Any) -> None: """Add a new value to the tracker.""" before = len(self.unique_set) - self._accum(self.unique_set, value) + self.unique_set.add(value) self.change_series.append(len(self.unique_set) > before) def classify(self) -> Classification: @@ -69,7 +79,8 @@ def to_state(self) -> Dict[str, Any]: "type": self.__class__.__name__, "module": self.__class__.__module__, "min_samples": self.min_samples, - "expand_value": self.expand_value, + "add_value_fn": self.add_value_fn, + "detector_config": self.detector_config, "runs": self.change_series.runs(), "unique_set": list(self.unique_set), "segment_thresholds": self.stability_classifier.segment_threshs, @@ -81,7 +92,8 @@ def from_state(cls, state: Dict[str, Any]) -> "SingleStabilityTracker": """Restore tracker from a state dict produced by to_state().""" tracker = cls( min_samples=state["min_samples"], - expand_value=state.get("expand_value", False), + add_value_fn=state["add_value_fn"], + detector_config=state["detector_config"] ) runs = [(bool(r[0]), int(r[1])) for r in state["runs"]] tracker.change_series._runs = runs @@ -131,12 +143,14 @@ class EventStabilityTracker(EventTracker): def __init__( self, converter_function: Callable[[Any], Any] = lambda x: x, - expand_value: bool = False, + add_value_fn: str = "default", + detector_config: "CoreDetectorConfig | None" = None + ) -> None: self.multi_tracker: MultiStabilityTracker # for type hinting def make_tracker() -> SingleStabilityTracker: - return SingleStabilityTracker(expand_value=expand_value) + return SingleStabilityTracker(add_value_fn=add_value_fn, detector_config=detector_config) # Mirror class identity onto the closure so dump()/load() can resolve # the underlying SingleStabilityTracker via its module + qualname. diff --git a/state/CharsetDetector/metadata.json b/state/CharsetDetector/metadata.json new file mode 100644 index 00000000..fdeae5bf --- /dev/null +++ b/state/CharsetDetector/metadata.json @@ -0,0 +1,11 @@ +{ + "version": 1, + "saved_at": "2026-06-08T09:51:52.538329+00:00", + "events_seen": [], + "event_templates": {}, + "event_backends": {}, + "event_extensions": {}, + "event_data_kwargs": { + "add_value_fn": "CharsetDetector" + } +}