From 895a0d46791d61d7ab14efedd3ae0c6bd8c8393c Mon Sep 17 00:00:00 2001 From: Ernst Leierzopf Date: Mon, 8 Jun 2026 09:51:18 +0200 Subject: [PATCH 1/7] Implement custom add_value methods with add_value_fn argument in EventStabilityTracker. --- .../test_bigram_frequency_detector.py | 4 +- .../test_detectors/test_charset_detector.py | 14 +---- .../detectors/bigram_frequency_detector.py | 61 +++++++++++++------ .../detectors/charset_detector.py | 19 ++++-- .../detectors/value_range_detector.py | 43 +++++++++---- .../trackers/stability/stability_tracker.py | 18 +++--- 6 files changed, 99 insertions(+), 60 deletions(-) diff --git a/detectmatelibrary_tests/test_detectors/test_bigram_frequency_detector.py b/detectmatelibrary_tests/test_detectors/test_bigram_frequency_detector.py index 3e678d47..22937cee 100644 --- a/detectmatelibrary_tests/test_detectors/test_bigram_frequency_detector.py +++ b/detectmatelibrary_tests/test_detectors/test_bigram_frequency_detector.py @@ -255,7 +255,7 @@ def test_audit_log_anomalies(self): if detector.detect(log, output_=output): detected_ids.add(log["logID"]) - assert detected_ids == {'1859', '1860', '1861', '1862', '1864', '1865', '1866', '1867'} + assert detected_ids == {'1859', '1860', '1861', '1862'} class TestBigramFrequencyDetectorAutoConfig: @@ -288,7 +288,7 @@ def test_audit_log_anomalies_via_process(self): if detector.process(log) is not None: detected_ids.add(log["logID"]) - assert detected_ids == {'1859', '1860', '1861', '1862', '1864', '1865', '1866', '1867'} + assert detected_ids == {'1859', '1860', '1861', '1862'} class TestBigramFrequencyDetectorGlobalInstances: diff --git a/detectmatelibrary_tests/test_detectors/test_charset_detector.py b/detectmatelibrary_tests/test_detectors/test_charset_detector.py index d88fdfdd..8fe52531 100644 --- a/detectmatelibrary_tests/test_detectors/test_charset_detector.py +++ b/detectmatelibrary_tests/test_detectors/test_charset_detector.py @@ -83,7 +83,7 @@ def test_custom_config_initialization(self): assert hasattr(detector, 'persistency') assert isinstance(detector.persistency.events_data, dict) - def test_persistency_uses_expand_value(self): + def test_persistency_uses_custom_add_value(self): """Main persistency must accumulate characters; auto_conf must not.""" detector = CharsetDetector() # Ingest a sample so a SingleStabilityTracker is materialized @@ -93,20 +93,8 @@ def test_persistency_uses_expand_value(self): named_variables={"v": "hello"}, ) single = detector.persistency.get_event_data(1)["v"] - assert single.expand_value is True assert single.unique_set == {"h", "e", "l", "o"} - def test_auto_conf_persistency_does_not_expand(self): - detector = CharsetDetector() - detector.auto_conf_persistency.ingest_event( - event_id=1, - event_template="t", - named_variables={"v": "hello"}, - ) - single = detector.auto_conf_persistency.get_event_data(1)["v"] - assert single.expand_value is False - assert single.unique_set == {"hello"} - def test_register_persistency_was_called(self): """Main persistency should be registered so persist/load round-trips work.""" diff --git a/src/detectmatelibrary/detectors/bigram_frequency_detector.py b/src/detectmatelibrary/detectors/bigram_frequency_detector.py index 4daf163a..03cd4711 100644 --- a/src/detectmatelibrary/detectors/bigram_frequency_detector.py +++ b/src/detectmatelibrary/detectors/bigram_frequency_detector.py @@ -1,5 +1,4 @@ from typing import Any, cast - from detectmatelibrary.common._config._compile import generate_detector_config from detectmatelibrary.common._config._formats import EventsConfig from detectmatelibrary.common.detector import ( @@ -71,14 +70,50 @@ def __init__( if isinstance(config, dict): config = BigramFrequencyDetectorConfig.from_dict(config, name) + def add_value(cls: SingleStabilityTracker, value: Any) -> None: + """Add a new value to the tracker.""" + change = False + default_freq, default_total = (_default_freq_tables() if self.config.default_freqs else ({}, {})) + freq: dict[Any, dict[Any, int]] = cls.extra_state.get("freq", {}) + total_freq: dict[Any, int] = cls.extra_state.get("total_freq", {}) + probs: list[float] = [] + for i in range(-1, len(value)): + first: Any = -1 if i == -1 else value[i] + second: Any = -1 if i == len(value) - 1 else value[i + 1] + prob = 0.0 + if first in freq and second in freq[first] and total_freq.get(first, 0) > 0: + prob = freq[first][second] / total_freq[first] + elif self.config.default_freqs: + if (first in default_freq and second in default_freq[first] + and default_total.get(first, 0) > 0): + prob = default_freq[first][second] / default_total[first] + probs.append(prob) + if probs: + critical_val = sum(probs) / len(probs) + change = critical_val > self.config.prob_thresh or any(x == 0.0 for x in probs) + + if self.config.skip_repetitions and value in cls.unique_set: + change = False + else: + for i in range(-1, len(value)): + first = -1 if i == -1 else value[i] + second = -1 if i == len(value) - 1 else value[i + 1] + row = freq.setdefault(first, {}) + row[second] = row.get(second, 0) + 1 + total_freq[first] = total_freq.get(first, 0) + 1 + cls.unique_set.add(value) + cls.change_series.append(change) + super().__init__(name=name, buffer_mode=BufferMode.NO_BUF, config=config) self.config: BigramFrequencyDetectorConfig # type narrowing for IDE self.persistency = EventPersistency( event_data_class=EventStabilityTracker, + event_data_kwargs={"add_value_fn": add_value} ) - # auto config checks if individual variables are stable to select combos from + # auto config checks if individual variables are stable self.auto_conf_persistency = EventPersistency( - event_data_class=EventStabilityTracker + event_data_class=EventStabilityTracker, + event_data_kwargs={"add_value_fn": add_value} ) self._register_persistency(self.persistency) @@ -99,17 +134,13 @@ def train(self, input_: ParserSchema) -> None: # type: ignore named_variables=configured_variables, ) if configured_variables: - known_events = cast( - dict[int | str, EventStabilityTracker], self.persistency.get_events_data() - ) + known_events = cast(dict[int | str, EventStabilityTracker], self.persistency.get_events_data()) self.train_helper(configured_variables, current_event_id, known_events, pre_unique) if self.config.global_instances: global_vars = get_global_variables(input_, self.config.global_instances) if global_vars: - pre_unique_global = self._snapshot_unique_sets( - known_events.get(GLOBAL_EVENT_ID), global_vars - ) + pre_unique_global = self._snapshot_unique_sets(known_events.get(GLOBAL_EVENT_ID), global_vars) self.persistency.ingest_event( event_id=GLOBAL_EVENT_ID, event_template=input_["template"], @@ -175,9 +206,7 @@ def detect( configured_variables = get_configured_variables(input_, self.config.events) overall_score = 0.0 current_event_id = input_["EventID"] - known_events = cast( - dict[int | str, EventStabilityTracker], self.persistency.get_events_data() - ) + known_events = cast(dict[int | str, EventStabilityTracker], self.persistency.get_events_data()) if current_event_id in known_events: overall_score = self.detect_helper( alerts, configured_variables, current_event_id, known_events, overall_score @@ -203,12 +232,8 @@ def detect_helper( overall_score: float, ) -> float: anomaly = False - default_freq, default_total = ( - _default_freq_tables() if self.config.default_freqs else ({}, {}) - ) - var_trackers = cast( - dict[str, SingleStabilityTracker], known_events[event_id].get_data() - ) + default_freq, default_total = (_default_freq_tables() if self.config.default_freqs else ({}, {})) + var_trackers = cast(dict[str, SingleStabilityTracker], known_events[event_id].get_data()) for var_name, single_tracker in var_trackers.items(): value: Any = variables.get(var_name) if value is None: diff --git a/src/detectmatelibrary/detectors/charset_detector.py b/src/detectmatelibrary/detectors/charset_detector.py index d3b48f5d..e84e08e1 100644 --- a/src/detectmatelibrary/detectors/charset_detector.py +++ b/src/detectmatelibrary/detectors/charset_detector.py @@ -1,3 +1,4 @@ +from typing import Any from detectmatelibrary.common._config._compile import generate_detector_config from detectmatelibrary.common._config._formats import EventsConfig from detectmatelibrary.common.detector import ( @@ -8,7 +9,8 @@ validate_config_coverage, ) from detectmatelibrary.utils.persistency.event_data_structures.trackers.stability.stability_tracker import ( - EventStabilityTracker + EventStabilityTracker, + SingleStabilityTracker ) from detectmatelibrary.utils.persistency.event_persistency import EventPersistency from detectmatelibrary.utils.data_buffer import BufferMode @@ -37,14 +39,23 @@ def __init__( if isinstance(config, dict): config = CharsetDetectorConfig.from_dict(config, name) + def add_value(cls: SingleStabilityTracker, value: Any) -> None: + """Add a new value to the tracker.""" + before = len(cls.unique_set) + cls.unique_set.update(value) + cls.change_series.append(len(cls.unique_set) > before) + super().__init__(name=name, buffer_mode=BufferMode.NO_BUF, config=config) self.config: CharsetDetectorConfig # type narrowing for IDE self.persistency = EventPersistency( event_data_class=EventStabilityTracker, - event_data_kwargs={"expand_value": True}, + event_data_kwargs={"add_value_fn": add_value} + ) + # auto config checks if individual variables are stable to select characters from + self.auto_conf_persistency = EventPersistency( + event_data_class=EventStabilityTracker, + event_data_kwargs={"add_value_fn": add_value} ) - # auto config checks if individual variables are stable to select combos from - self.auto_conf_persistency = EventPersistency(event_data_class=EventStabilityTracker) self._register_persistency(self.persistency) def train(self, input_: ParserSchema) -> None: # type: ignore diff --git a/src/detectmatelibrary/detectors/value_range_detector.py b/src/detectmatelibrary/detectors/value_range_detector.py index bc3a0815..868e12e2 100644 --- a/src/detectmatelibrary/detectors/value_range_detector.py +++ b/src/detectmatelibrary/detectors/value_range_detector.py @@ -8,7 +8,8 @@ validate_config_coverage, ) from detectmatelibrary.utils.persistency.event_data_structures.trackers.stability.stability_tracker import ( - EventStabilityTracker + EventStabilityTracker, + SingleStabilityTracker ) from detectmatelibrary.utils.persistency.event_persistency import EventPersistency from detectmatelibrary.utils.data_buffer import BufferMode @@ -39,14 +40,31 @@ def __init__( if isinstance(config, dict): config = ValueRangeDetectorConfig.from_dict(config, name) + def add_value(cls: SingleStabilityTracker, value: int | float) -> None: + """Add a new value to the tracker.""" + try: + value = float(value) + value = int(value) if value.is_integer() else value + except ValueError: + return + if len(cls.unique_set) > 0: + min_ = min(cls.unique_set) + max_ = max(cls.unique_set) + cls.change_series.append(value < min_ or value > max_) + else: + cls.change_series.append(True) + cls.unique_set.add(value) + super().__init__(name=name, buffer_mode=BufferMode.NO_BUF, config=config) self.config: ValueRangeDetectorConfig # type narrowing for IDE self.persistency = EventPersistency( event_data_class=EventStabilityTracker, + event_data_kwargs={"add_value_fn": add_value} ) - # auto config checks if individual variables are stable to select combos from + # auto config checks if individual variables are stable to select value ranges from self.auto_conf_persistency = EventPersistency( - event_data_class=EventStabilityTracker + event_data_class=EventStabilityTracker, + event_data_kwargs={"add_value_fn": add_value} ) def cast_val_to_numeric(self, configured_variables: Dict[str, Any], k: str, remove: List[str], @@ -54,17 +72,16 @@ def cast_val_to_numeric(self, configured_variables: Dict[str, Any], k: str, remo v = configured_variables[k] if not isinstance(v, (int, float)): try: - configured_variables[k] = int(v) + configured_variables[k] = float(v) + configured_variables[k] = int(configured_variables[k])\ + if configured_variables[k].is_integer() else configured_variables[k] except ValueError: - try: - configured_variables[k] = float(v) - except ValueError: - logger.error(f"Non-numeric value '{v}' appeared in {stage} of {self.__class__.__name__}" - f" with the name {self.name}.") - if not self.config.ignore_non_numerical_val: - exit(1) - remove.append(k) - return False + logger.error(f"Non-numeric value '{v}' appeared in {stage} of {self.__class__.__name__}" + f" with the name {self.name}.") + if not self.config.ignore_non_numerical_val: + exit(1) + remove.append(k) + return False return True def train(self, input_: ParserSchema) -> None: # type: ignore diff --git a/src/detectmatelibrary/utils/persistency/event_data_structures/trackers/stability/stability_tracker.py b/src/detectmatelibrary/utils/persistency/event_data_structures/trackers/stability/stability_tracker.py index 8184cc0a..904583f3 100644 --- a/src/detectmatelibrary/utils/persistency/event_data_structures/trackers/stability/stability_tracker.py +++ b/src/detectmatelibrary/utils/persistency/event_data_structures/trackers/stability/stability_tracker.py @@ -1,10 +1,8 @@ """Tracks whether a variable is converging to a constant value.""" from typing import Any, Callable, Dict, List, Literal, Set - from detectmatelibrary.utils.preview_helpers import list_preview_str from detectmatelibrary.utils.persistency.rle_list import RLEList - from ..base import SingleTracker, MultiTracker, EventTracker, Classification from .stability_classifier import StabilityClassifier @@ -12,23 +10,24 @@ class SingleStabilityTracker(SingleTracker): """Tracks stability of a single feature.""" - def __init__(self, min_samples: int = 3, expand_value: bool = False) -> None: + def __init__(self, min_samples: int = 3, add_value_fn: Callable[..., None] | None = None) -> None: self.min_samples = min_samples - self.expand_value = expand_value self.change_series: RLEList[bool] = RLEList() self.unique_set: Set[Any] = set() self.stability_classifier: StabilityClassifier = StabilityClassifier( segment_thresholds=[1.1, 0.3, 0.1, 0.01], ) - self._accum = set.update if expand_value else set.add # Opaque slot for detectors to stash per-variable model state that # must survive save/load. Schema-free; the tracker does not interpret it. self.extra_state: Dict[str, Any] = {} + self.add_value_fn = add_value_fn + if add_value_fn is not None: + self.add_value = add_value_fn.__get__(self, type(self)) # type: ignore[method-assign] def add_value(self, value: Any) -> None: """Add a new value to the tracker.""" before = len(self.unique_set) - self._accum(self.unique_set, value) + self.unique_set.add(value) self.change_series.append(len(self.unique_set) > before) def classify(self) -> Classification: @@ -69,7 +68,6 @@ def to_state(self) -> Dict[str, Any]: "type": self.__class__.__name__, "module": self.__class__.__module__, "min_samples": self.min_samples, - "expand_value": self.expand_value, "runs": self.change_series.runs(), "unique_set": list(self.unique_set), "segment_thresholds": self.stability_classifier.segment_threshs, @@ -81,7 +79,7 @@ def from_state(cls, state: Dict[str, Any]) -> "SingleStabilityTracker": """Restore tracker from a state dict produced by to_state().""" tracker = cls( min_samples=state["min_samples"], - expand_value=state.get("expand_value", False), + add_value_fn=cls.add_value, ) runs = [(bool(r[0]), int(r[1])) for r in state["runs"]] tracker.change_series._runs = runs @@ -131,12 +129,12 @@ class EventStabilityTracker(EventTracker): def __init__( self, converter_function: Callable[[Any], Any] = lambda x: x, - expand_value: bool = False, + add_value_fn: Callable[..., None] | None = None ) -> None: self.multi_tracker: MultiStabilityTracker # for type hinting def make_tracker() -> SingleStabilityTracker: - return SingleStabilityTracker(expand_value=expand_value) + return SingleStabilityTracker(add_value_fn=add_value_fn) # Mirror class identity onto the closure so dump()/load() can resolve # the underlying SingleStabilityTracker via its module + qualname. From 7bf25ccd2436e7547b18ce78177705c3cec494d9 Mon Sep 17 00:00:00 2001 From: Ernst Leierzopf Date: Mon, 8 Jun 2026 15:26:06 +0200 Subject: [PATCH 2/7] fix stability_tracker and unittests. --- .../test_stability_tracker.py | 41 +++++++++++-------- src/detectmatelibrary/detectors/__init__.py | 5 ++- .../detectors/bigram_frequency_detector.py | 9 ++-- .../detectors/charset_detector.py | 7 +++- .../detectors/value_range_detector.py | 7 +++- .../trackers/stability/stability_tracker.py | 30 ++++++++++---- state/CharsetDetector/metadata.json | 11 +++++ 7 files changed, 77 insertions(+), 33 deletions(-) create mode 100644 state/CharsetDetector/metadata.json diff --git a/detectmatelibrary_tests/test_persistency/test_stability_tracker.py b/detectmatelibrary_tests/test_persistency/test_stability_tracker.py index d6aa34a6..894c9888 100644 --- a/detectmatelibrary_tests/test_persistency/test_stability_tracker.py +++ b/detectmatelibrary_tests/test_persistency/test_stability_tracker.py @@ -5,32 +5,32 @@ class TestSingleStabilityTrackerExpandValue: - def test_default_add_stores_whole_value(self): + def test_default_add_value(self): tracker = SingleStabilityTracker() tracker.add_value("hello") tracker.add_value("world") assert tracker.unique_set == {"hello", "world"} - def test_expand_value_unions_characters(self): - tracker = SingleStabilityTracker(expand_value=True) + def test_custom_add_value(self): + tracker = SingleStabilityTracker(add_value_fn="CharsetDetector") tracker.add_value("hello") tracker.add_value("world") assert tracker.unique_set == {"h", "e", "l", "o", "w", "r", "d"} def test_expand_value_change_series_tracks_growth(self): - tracker = SingleStabilityTracker(expand_value=True) + tracker = SingleStabilityTracker(add_value_fn="CharsetDetector") tracker.add_value("ab") # adds {a, b}, change=True tracker.add_value("ba") # adds nothing new, change=False tracker.add_value("c") # adds {c}, change=True assert list(tracker.change_series) == [True, False, True] def test_expand_value_round_trip(self): - tracker = SingleStabilityTracker(expand_value=True) + tracker = SingleStabilityTracker(add_value_fn="CharsetDetector") tracker.add_value("hello") tracker.add_value("world") state = tracker.to_state() restored = SingleStabilityTracker.from_state(state) - assert restored.expand_value is True + assert restored.add_value_fn == "CharsetDetector" assert restored.unique_set == {"h", "e", "l", "o", "w", "r", "d"} # subsequent ingestion still unions characters restored.add_value("xy") @@ -40,9 +40,8 @@ def test_legacy_state_without_expand_value_defaults_false(self): tracker = SingleStabilityTracker() tracker.add_value("hello") state = tracker.to_state() - state.pop("expand_value", None) # simulate pre-flag snapshot restored = SingleStabilityTracker.from_state(state) - assert restored.expand_value is False + assert restored.add_value_fn == "default" assert restored.unique_set == {"hello"} @@ -51,45 +50,51 @@ def test_default_event_tracker_uses_add_semantics(self): event_tracker = EventStabilityTracker() event_tracker.add_data({"var1": "hello"}) single = event_tracker.get_data()["var1"] - assert single.expand_value is False + assert single.add_value_fn == "default" + assert single.detector_config is None assert single.unique_set == {"hello"} def test_expand_value_propagates_to_per_variable_trackers(self): - event_tracker = EventStabilityTracker(expand_value=True) + event_tracker = EventStabilityTracker(add_value_fn="CharsetDetector") event_tracker.add_data({"var1": "hello"}) event_tracker.add_data({"var1": "world"}) single = event_tracker.get_data()["var1"] - assert single.expand_value is True + assert single.add_value_fn == "CharsetDetector" + assert single.detector_config is None assert single.unique_set == {"h", "e", "l", "o", "w", "r", "d"} def test_each_new_variable_gets_its_own_configured_tracker(self): - event_tracker = EventStabilityTracker(expand_value=True) + event_tracker = EventStabilityTracker(add_value_fn="CharsetDetector") event_tracker.add_data({"a": "ab", "b": "cd"}) a = event_tracker.get_data()["a"] b = event_tracker.get_data()["b"] - assert a.expand_value is True - assert b.expand_value is True + assert a.add_value_fn == "CharsetDetector" + assert b.add_value_fn == "CharsetDetector" + assert a.detector_config is None + assert b.detector_config is None assert a.unique_set == {"a", "b"} assert b.unique_set == {"c", "d"} def test_post_load_new_variable_honors_expand_value(self): """After dump/load, a variable that wasn't present at save time should still use expand_value semantics when first ingested.""" - original = EventStabilityTracker(expand_value=True) + original = EventStabilityTracker(add_value_fn="CharsetDetector") original.add_data({"known": "abc"}) blob = original.dump() - restored = EventStabilityTracker.load(blob, expand_value=True) + restored = EventStabilityTracker.load(blob, add_value_fn="CharsetDetector") # Ingest a brand-new variable not present in the saved state restored.add_data({"known": "de", "newvar": "xy"}) new_tracker = restored.get_data()["newvar"] - assert new_tracker.expand_value is True + assert new_tracker.add_value_fn == "CharsetDetector" + assert new_tracker.detector_config is None assert new_tracker.unique_set == {"x", "y"} # And the existing variable continues to expand correctly known_tracker = restored.get_data()["known"] - assert known_tracker.expand_value is True + assert known_tracker.add_value_fn == "CharsetDetector" + assert known_tracker.detector_config is None assert {"a", "b", "c", "d", "e"} <= known_tracker.unique_set diff --git a/src/detectmatelibrary/detectors/__init__.py b/src/detectmatelibrary/detectors/__init__.py index 361db003..9fa88739 100644 --- a/src/detectmatelibrary/detectors/__init__.py +++ b/src/detectmatelibrary/detectors/__init__.py @@ -1,3 +1,4 @@ +from .bigram_frequency_detector import BigramFrequencyDetector, BigramFrequencyDetectorConfig from .random_detector import RandomDetector, RandomDetectorConfig from .new_value_detector import NewValueDetector, NewValueDetectorConfig from .new_event_detector import NewEventDetector, NewEventDetectorConfig @@ -15,5 +16,7 @@ "ValueRangeDetector", "ValueRangeDetectorConfig", "CharsetDetector", - "CharsetDetectorConfig" + "CharsetDetectorConfig", + "BigramFrequencyDetector", + "BigramFrequencyDetectorConfig" ] diff --git a/src/detectmatelibrary/detectors/bigram_frequency_detector.py b/src/detectmatelibrary/detectors/bigram_frequency_detector.py index 03cd4711..934bf880 100644 --- a/src/detectmatelibrary/detectors/bigram_frequency_detector.py +++ b/src/detectmatelibrary/detectors/bigram_frequency_detector.py @@ -66,7 +66,6 @@ def __init__( name: str = "BigramFrequencyDetector", config: BigramFrequencyDetectorConfig = BigramFrequencyDetectorConfig() ) -> None: - if isinstance(config, dict): config = BigramFrequencyDetectorConfig.from_dict(config, name) @@ -103,17 +102,21 @@ def add_value(cls: SingleStabilityTracker, value: Any) -> None: total_freq[first] = total_freq.get(first, 0) + 1 cls.unique_set.add(value) cls.change_series.append(change) + self.add_value_fn = add_value super().__init__(name=name, buffer_mode=BufferMode.NO_BUF, config=config) self.config: BigramFrequencyDetectorConfig # type narrowing for IDE + kwargs = {"add_value_fn": self.__class__.__name__, "detector_config": self.config.to_dict( + method_id="BigramFrequencyDetector")} self.persistency = EventPersistency( event_data_class=EventStabilityTracker, - event_data_kwargs={"add_value_fn": add_value} + event_data_kwargs=kwargs + ) # auto config checks if individual variables are stable self.auto_conf_persistency = EventPersistency( event_data_class=EventStabilityTracker, - event_data_kwargs={"add_value_fn": add_value} + event_data_kwargs=kwargs ) self._register_persistency(self.persistency) diff --git a/src/detectmatelibrary/detectors/charset_detector.py b/src/detectmatelibrary/detectors/charset_detector.py index e84e08e1..13edea1c 100644 --- a/src/detectmatelibrary/detectors/charset_detector.py +++ b/src/detectmatelibrary/detectors/charset_detector.py @@ -44,17 +44,20 @@ def add_value(cls: SingleStabilityTracker, value: Any) -> None: before = len(cls.unique_set) cls.unique_set.update(value) cls.change_series.append(len(cls.unique_set) > before) + self.add_value_fn = add_value super().__init__(name=name, buffer_mode=BufferMode.NO_BUF, config=config) self.config: CharsetDetectorConfig # type narrowing for IDE + kwargs = {"add_value_fn": self.__class__.__name__, "detector_config": self.config.to_dict( + method_id="CharsetDetector")} self.persistency = EventPersistency( event_data_class=EventStabilityTracker, - event_data_kwargs={"add_value_fn": add_value} + event_data_kwargs=kwargs ) # auto config checks if individual variables are stable to select characters from self.auto_conf_persistency = EventPersistency( event_data_class=EventStabilityTracker, - event_data_kwargs={"add_value_fn": add_value} + event_data_kwargs=kwargs ) self._register_persistency(self.persistency) diff --git a/src/detectmatelibrary/detectors/value_range_detector.py b/src/detectmatelibrary/detectors/value_range_detector.py index 868e12e2..67e54159 100644 --- a/src/detectmatelibrary/detectors/value_range_detector.py +++ b/src/detectmatelibrary/detectors/value_range_detector.py @@ -54,17 +54,20 @@ def add_value(cls: SingleStabilityTracker, value: int | float) -> None: else: cls.change_series.append(True) cls.unique_set.add(value) + self.add_value_fn = add_value super().__init__(name=name, buffer_mode=BufferMode.NO_BUF, config=config) self.config: ValueRangeDetectorConfig # type narrowing for IDE + kwargs = {"add_value_fn": self.__class__.__name__, "detector_config": self.config.to_dict( + method_id="ValueRangeDetector")} self.persistency = EventPersistency( event_data_class=EventStabilityTracker, - event_data_kwargs={"add_value_fn": add_value} + event_data_kwargs=kwargs ) # auto config checks if individual variables are stable to select value ranges from self.auto_conf_persistency = EventPersistency( event_data_class=EventStabilityTracker, - event_data_kwargs={"add_value_fn": add_value} + event_data_kwargs=kwargs ) def cast_val_to_numeric(self, configured_variables: Dict[str, Any], k: str, remove: List[str], diff --git a/src/detectmatelibrary/utils/persistency/event_data_structures/trackers/stability/stability_tracker.py b/src/detectmatelibrary/utils/persistency/event_data_structures/trackers/stability/stability_tracker.py index 904583f3..c32ff8ac 100644 --- a/src/detectmatelibrary/utils/persistency/event_data_structures/trackers/stability/stability_tracker.py +++ b/src/detectmatelibrary/utils/persistency/event_data_structures/trackers/stability/stability_tracker.py @@ -1,16 +1,21 @@ """Tracks whether a variable is converging to a constant value.""" -from typing import Any, Callable, Dict, List, Literal, Set +import importlib +from typing import Any, Callable, Dict, List, Literal, Set, TYPE_CHECKING from detectmatelibrary.utils.preview_helpers import list_preview_str from detectmatelibrary.utils.persistency.rle_list import RLEList from ..base import SingleTracker, MultiTracker, EventTracker, Classification from .stability_classifier import StabilityClassifier +if TYPE_CHECKING: + from detectmatelibrary.common.detector import CoreDetectorConfig + class SingleStabilityTracker(SingleTracker): """Tracks stability of a single feature.""" - def __init__(self, min_samples: int = 3, add_value_fn: Callable[..., None] | None = None) -> None: + def __init__(self, min_samples: int = 3, add_value_fn: str = "default", + detector_config: "CoreDetectorConfig | None" = None) -> None: self.min_samples = min_samples self.change_series: RLEList[bool] = RLEList() self.unique_set: Set[Any] = set() @@ -21,8 +26,14 @@ def __init__(self, min_samples: int = 3, add_value_fn: Callable[..., None] | Non # must survive save/load. Schema-free; the tracker does not interpret it. self.extra_state: Dict[str, Any] = {} self.add_value_fn = add_value_fn - if add_value_fn is not None: - self.add_value = add_value_fn.__get__(self, type(self)) # type: ignore[method-assign] + self.detector_config = detector_config + if add_value_fn != "default": + detector = getattr(importlib.import_module("detectmatelibrary.detectors"), add_value_fn) + if detector_config is not None: + detector = detector(config=detector_config) + else: + detector = detector() + self.add_value = detector.add_value_fn.__get__(self, type(self)) # type: ignore[method-assign] def add_value(self, value: Any) -> None: """Add a new value to the tracker.""" @@ -68,6 +79,8 @@ def to_state(self) -> Dict[str, Any]: "type": self.__class__.__name__, "module": self.__class__.__module__, "min_samples": self.min_samples, + "add_value_fn": self.add_value_fn, + "detector_config": self.detector_config, "runs": self.change_series.runs(), "unique_set": list(self.unique_set), "segment_thresholds": self.stability_classifier.segment_threshs, @@ -79,7 +92,8 @@ def from_state(cls, state: Dict[str, Any]) -> "SingleStabilityTracker": """Restore tracker from a state dict produced by to_state().""" tracker = cls( min_samples=state["min_samples"], - add_value_fn=cls.add_value, + add_value_fn=state["add_value_fn"], + detector_config=state["detector_config"] ) runs = [(bool(r[0]), int(r[1])) for r in state["runs"]] tracker.change_series._runs = runs @@ -129,12 +143,14 @@ class EventStabilityTracker(EventTracker): def __init__( self, converter_function: Callable[[Any], Any] = lambda x: x, - add_value_fn: Callable[..., None] | None = None + add_value_fn: str = "default", + detector_config: "CoreDetectorConfig | None" = None + ) -> None: self.multi_tracker: MultiStabilityTracker # for type hinting def make_tracker() -> SingleStabilityTracker: - return SingleStabilityTracker(add_value_fn=add_value_fn) + return SingleStabilityTracker(add_value_fn=add_value_fn, detector_config=detector_config) # Mirror class identity onto the closure so dump()/load() can resolve # the underlying SingleStabilityTracker via its module + qualname. diff --git a/state/CharsetDetector/metadata.json b/state/CharsetDetector/metadata.json new file mode 100644 index 00000000..b2f416fe --- /dev/null +++ b/state/CharsetDetector/metadata.json @@ -0,0 +1,11 @@ +{ + "version": 1, + "saved_at": "2026-06-08T09:51:52.538329+00:00", + "events_seen": [], + "event_templates": {}, + "event_backends": {}, + "event_extensions": {}, + "event_data_kwargs": { + "add_value_fn": "CharsetDetector" + } +} \ No newline at end of file From 9ac644768ecdc7129942a67697340ebcb379c144 Mon Sep 17 00:00:00 2001 From: Ernst Leierzopf Date: Mon, 8 Jun 2026 15:28:01 +0200 Subject: [PATCH 3/7] fix prek. --- state/CharsetDetector/metadata.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/state/CharsetDetector/metadata.json b/state/CharsetDetector/metadata.json index b2f416fe..fdeae5bf 100644 --- a/state/CharsetDetector/metadata.json +++ b/state/CharsetDetector/metadata.json @@ -8,4 +8,4 @@ "event_data_kwargs": { "add_value_fn": "CharsetDetector" } -} \ No newline at end of file +} From ccf45dc0779a19bd90af1ff05b6f31854aef911d Mon Sep 17 00:00:00 2001 From: Ernst Leierzopf Date: Mon, 6 Jul 2026 17:57:39 +0200 Subject: [PATCH 4/7] fix conflicts. --- .pre-commit-config.yaml | 4 ++-- AGENTS.md | 2 +- config/pipeline_config_default.yaml | 2 +- docs/parsers/template_matcher.md | 2 +- docs/parsers/template_tree_matcher.md | 2 +- .../workspace/create_workspace.py | 6 +++--- .../__init__.py | 0 .../test_common/__init__.py | 0 .../test_common/test_config.py | 2 +- .../test_common/test_config_roundtrip.py | 2 +- .../test_common/test_core.py | 0 .../test_common/test_core_detector.py | 0 .../test_common/test_core_parser.py | 0 .../test_common/test_extract_timestamp.py | 0 .../test_common/test_fit_logic.py | 0 .../test_common/test_persist_config.py | 0 .../test_detectors/__init__.py | 0 .../test_detectors/dummy_detector.py | 0 .../test_bigram_frequency_detector.py | 4 ++-- .../test_detectors/test_charset_detector.py | 4 ++-- .../test_detectors/test_mismatch_warnings.py | 0 .../test_detectors/test_new_event_detector.py | 4 ++-- .../test_new_value_combo_detector.py | 4 ++-- .../test_detectors/test_new_value_detector.py | 4 ++-- .../test_persist_integration.py | 0 .../test_detectors/test_random_detector.py | 0 .../test_detectors/test_rule_detector.py | 0 .../test_value_range_detector.py | 4 ++-- .../test_folder/__init__.py | 0 .../test_folder/audit.log | 0 .../test_folder/audit_anomaly_labels.log | 0 .../test_folder/audit_templates.txt | 0 .../test_folder/logs.log | 0 .../test_folder/test_config.yaml | 2 +- .../test_folder/test_named_templates.csv | 0 .../test_folder/test_named_templates.txt | 0 .../test_folder/test_templates.txt | 0 .../test_helper/__init__.py | 0 .../test_helper/test_from_to.py | 20 +++++++++---------- .../test_others/__init__.py | 0 .../test_others/test_logging.py | 0 .../test_parsers/__init__.py | 0 .../test_parsers/dummy_parser.py | 0 .../test_parsers/test_json_parser.py | 2 +- .../test_parsers/test_logbatcher_parser.py | 0 .../test_parsers/test_template_matcher.py | 10 +++++----- .../test_parsers/test_tree_macther.py | 2 +- .../test_persistency/__init__.py | 0 .../test_stability_tracker.py | 0 .../test_pipelines/__init__.py | 0 .../test_pipelines/test_bad_players.py | 2 +- .../test_pipelines/test_basic_pipelines.py | 4 ++-- .../test_configuration_engine.py | 6 +++--- .../test_pipelines/test_dummy_methods.py | 4 ++-- .../test_schemas/__init__.py | 0 .../test_schemas/test_default_values.py | 0 .../test_schemas/test_ops.py | 0 .../test_schemas/test_schema_class.py | 0 .../test_utils/__init__.py | 0 .../test_utils/test_aux.py | 0 .../test_utils/test_data_buffer.py | 0 .../test_utils/test_data_normalizer.py | 0 .../test_utils/test_events_seen.py | 0 .../test_utils/test_log_format_utils.py | 0 .../test_utils/test_persistency.py | 0 .../test_utils/test_persistency_dump_load.py | 0 .../test_utils/test_persistency_saver.py | 0 .../test_utils/test_stability_tracking.py | 0 .../test_workspace/__init__.py | 0 .../test_workspace/test_create_workspace.py | 10 +++++----- .../test_workspace/test_utils.py | 0 71 files changed, 54 insertions(+), 54 deletions(-) rename {detectmatelibrary_tests => tests}/__init__.py (100%) rename {detectmatelibrary_tests => tests}/test_common/__init__.py (100%) rename {detectmatelibrary_tests => tests}/test_common/test_config.py (98%) rename {detectmatelibrary_tests => tests}/test_common/test_config_roundtrip.py (99%) rename {detectmatelibrary_tests => tests}/test_common/test_core.py (100%) rename {detectmatelibrary_tests => tests}/test_common/test_core_detector.py (100%) rename {detectmatelibrary_tests => tests}/test_common/test_core_parser.py (100%) rename {detectmatelibrary_tests => tests}/test_common/test_extract_timestamp.py (100%) rename {detectmatelibrary_tests => tests}/test_common/test_fit_logic.py (100%) rename {detectmatelibrary_tests => tests}/test_common/test_persist_config.py (100%) rename {detectmatelibrary_tests => tests}/test_detectors/__init__.py (100%) rename {detectmatelibrary_tests => tests}/test_detectors/dummy_detector.py (100%) rename {detectmatelibrary_tests => tests}/test_detectors/test_bigram_frequency_detector.py (99%) rename {detectmatelibrary_tests => tests}/test_detectors/test_charset_detector.py (98%) rename {detectmatelibrary_tests => tests}/test_detectors/test_mismatch_warnings.py (100%) rename {detectmatelibrary_tests => tests}/test_detectors/test_new_event_detector.py (97%) rename {detectmatelibrary_tests => tests}/test_detectors/test_new_value_combo_detector.py (99%) rename {detectmatelibrary_tests => tests}/test_detectors/test_new_value_detector.py (98%) rename {detectmatelibrary_tests => tests}/test_detectors/test_persist_integration.py (100%) rename {detectmatelibrary_tests => tests}/test_detectors/test_random_detector.py (100%) rename {detectmatelibrary_tests => tests}/test_detectors/test_rule_detector.py (100%) rename {detectmatelibrary_tests => tests}/test_detectors/test_value_range_detector.py (98%) rename {detectmatelibrary_tests => tests}/test_folder/__init__.py (100%) rename {detectmatelibrary_tests => tests}/test_folder/audit.log (100%) rename {detectmatelibrary_tests => tests}/test_folder/audit_anomaly_labels.log (100%) rename {detectmatelibrary_tests => tests}/test_folder/audit_templates.txt (100%) rename {detectmatelibrary_tests => tests}/test_folder/logs.log (100%) rename {detectmatelibrary_tests => tests}/test_folder/test_config.yaml (98%) rename {detectmatelibrary_tests => tests}/test_folder/test_named_templates.csv (100%) rename {detectmatelibrary_tests => tests}/test_folder/test_named_templates.txt (100%) rename {detectmatelibrary_tests => tests}/test_folder/test_templates.txt (100%) rename {detectmatelibrary_tests => tests}/test_helper/__init__.py (100%) rename {detectmatelibrary_tests => tests}/test_helper/test_from_to.py (95%) rename {detectmatelibrary_tests => tests}/test_others/__init__.py (100%) rename {detectmatelibrary_tests => tests}/test_others/test_logging.py (100%) rename {detectmatelibrary_tests => tests}/test_parsers/__init__.py (100%) rename {detectmatelibrary_tests => tests}/test_parsers/dummy_parser.py (100%) rename {detectmatelibrary_tests => tests}/test_parsers/test_json_parser.py (98%) rename {detectmatelibrary_tests => tests}/test_parsers/test_logbatcher_parser.py (100%) rename {detectmatelibrary_tests => tests}/test_parsers/test_template_matcher.py (95%) rename {detectmatelibrary_tests => tests}/test_parsers/test_tree_macther.py (94%) rename {detectmatelibrary_tests => tests}/test_persistency/__init__.py (100%) rename {detectmatelibrary_tests => tests}/test_persistency/test_stability_tracker.py (100%) rename {detectmatelibrary_tests => tests}/test_pipelines/__init__.py (100%) rename {detectmatelibrary_tests => tests}/test_pipelines/test_bad_players.py (96%) rename {detectmatelibrary_tests => tests}/test_pipelines/test_basic_pipelines.py (95%) rename {detectmatelibrary_tests => tests}/test_pipelines/test_configuration_engine.py (93%) rename {detectmatelibrary_tests => tests}/test_pipelines/test_dummy_methods.py (92%) rename {detectmatelibrary_tests => tests}/test_schemas/__init__.py (100%) rename {detectmatelibrary_tests => tests}/test_schemas/test_default_values.py (100%) rename {detectmatelibrary_tests => tests}/test_schemas/test_ops.py (100%) rename {detectmatelibrary_tests => tests}/test_schemas/test_schema_class.py (100%) rename {detectmatelibrary_tests => tests}/test_utils/__init__.py (100%) rename {detectmatelibrary_tests => tests}/test_utils/test_aux.py (100%) rename {detectmatelibrary_tests => tests}/test_utils/test_data_buffer.py (100%) rename {detectmatelibrary_tests => tests}/test_utils/test_data_normalizer.py (100%) rename {detectmatelibrary_tests => tests}/test_utils/test_events_seen.py (100%) rename {detectmatelibrary_tests => tests}/test_utils/test_log_format_utils.py (100%) rename {detectmatelibrary_tests => tests}/test_utils/test_persistency.py (100%) rename {detectmatelibrary_tests => tests}/test_utils/test_persistency_dump_load.py (100%) rename {detectmatelibrary_tests => tests}/test_utils/test_persistency_saver.py (100%) rename {detectmatelibrary_tests => tests}/test_utils/test_stability_tracking.py (100%) rename {detectmatelibrary_tests => tests}/test_workspace/__init__.py (100%) rename {detectmatelibrary_tests => tests}/test_workspace/test_create_workspace.py (95%) rename {detectmatelibrary_tests => tests}/test_workspace/test_utils.py (100%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 50901a6f..b651b0ed 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,7 +26,7 @@ repos: hooks: - id: mypy args: [--strict, --ignore-missing-imports] - exclude: ^(docs/|detectmatelibrary_tests/) + exclude: ^(docs/|tests/) additional_dependencies: - pydantic - types-PyYAML @@ -58,7 +58,7 @@ repos: rev: 1.8.6 hooks: - id: bandit - exclude: (^detectmatelibrary_tests/|.*/test_.*|^test_.*) + exclude: (^tests/|.*/test_.*|^test_.*) # Unused code detection - repo: https://github.com/jendrikseipp/vulture diff --git a/AGENTS.md b/AGENTS.md index b13c3dcb..dd994543 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -17,7 +17,7 @@ uv run prek install uv run pytest -q uv run pytest -s # verbose with stdout uv run pytest --cov=. --cov-report=term-missing # with coverage -uv run pytest detectmatelibrary_tests/test_foo.py # single test file +uv run pytest tests/test_foo.py # single test file # Run linting/formatting (all pre-commit hooks) uv run prek run -a diff --git a/config/pipeline_config_default.yaml b/config/pipeline_config_default.yaml index c9888068..92cbff0a 100644 --- a/config/pipeline_config_default.yaml +++ b/config/pipeline_config_default.yaml @@ -8,7 +8,7 @@ parsers: remove_spaces: True remove_punctuation: True lowercase: True - path_templates: detectmatelibrary_tests/test_folder/audit_templates.txt + path_templates: tests/test_folder/audit_templates.txt JsonMatcherParser: method_type: matcher_parser diff --git a/docs/parsers/template_matcher.md b/docs/parsers/template_matcher.md index 313190e3..84b56370 100644 --- a/docs/parsers/template_matcher.md +++ b/docs/parsers/template_matcher.md @@ -80,7 +80,7 @@ cfg = { "MatcherParser": { "method_type": "matcher_parser", "params": { - "path_templates": "detectmatelibrary_tests/test_folder/test_templates.txt", + "path_templates": "tests/test_folder/test_templates.txt", "remove_spaces": True, "remove_punctuation": True, "lowercase": True diff --git a/docs/parsers/template_tree_matcher.md b/docs/parsers/template_tree_matcher.md index afd89add..d62c3137 100644 --- a/docs/parsers/template_tree_matcher.md +++ b/docs/parsers/template_tree_matcher.md @@ -56,7 +56,7 @@ cfg = { "TreeMatcher": { "method_type": "tree_matcher", "params": { - "path_templates": "detectmatelibrary_tests/test_folder/test_templates.txt" + "path_templates": "tests/test_folder/test_templates.txt" } } } diff --git a/src/detectmatelibrary_tools/workspace/create_workspace.py b/src/detectmatelibrary_tools/workspace/create_workspace.py index 0d79a09d..bf569ada 100644 --- a/src/detectmatelibrary_tools/workspace/create_workspace.py +++ b/src/detectmatelibrary_tools/workspace/create_workspace.py @@ -33,7 +33,7 @@ def camelize(name: str) -> str: def create_tests(type_: str, name: str, workspace_root: Path, pkg_name: str) -> None: - """Create a detectmatelibrary_tests/ directory with a basic pytest file for + """Create a tests/ directory with a basic pytest file for the component. - Reads template tests from src/detectmatelibrary_tools/workspace/templates/test_templates/ @@ -41,7 +41,7 @@ def create_tests(type_: str, name: str, workspace_root: Path, pkg_name: str) -> - Renames CustomParser/CustomDetector to the camelized class name """ - tests_dir = workspace_root / "detectmatelibrary_tests" + tests_dir = workspace_root / "tests" tests_dir.mkdir(parents=True, exist_ok=True) template_file = TEMPLATE_DIR / "test_templates" / f"test_Custom{type_.capitalize()}.py" test_file = tests_dir / f"test_{name}.py" @@ -64,7 +64,7 @@ def create_tests(type_: str, name: str, workspace_root: Path, pkg_name: str) -> # replace the import line content = template_content.replace(original_import, new_import) # replace the remaining occurrences of CustomParser/CustomDetector - # with the new class name (inside the detectmatelibrary_tests) + # with the new class name (inside the tests) content = content.replace(base_class, new_class) content = content.replace(f'"custom_{type_}"', f'"{name}_{type_}"') content = content.rstrip() + "\n" diff --git a/detectmatelibrary_tests/__init__.py b/tests/__init__.py similarity index 100% rename from detectmatelibrary_tests/__init__.py rename to tests/__init__.py diff --git a/detectmatelibrary_tests/test_common/__init__.py b/tests/test_common/__init__.py similarity index 100% rename from detectmatelibrary_tests/test_common/__init__.py rename to tests/test_common/__init__.py diff --git a/detectmatelibrary_tests/test_common/test_config.py b/tests/test_common/test_config.py similarity index 98% rename from detectmatelibrary_tests/test_common/test_config.py rename to tests/test_common/test_config.py index bf7f3d14..7d53d97c 100644 --- a/detectmatelibrary_tests/test_common/test_config.py +++ b/tests/test_common/test_config.py @@ -19,7 +19,7 @@ def load_test_config() -> dict: - with open("detectmatelibrary_tests/test_folder/test_config.yaml", 'r') as file: + with open("tests/test_folder/test_config.yaml", 'r') as file: return yaml.safe_load(file) diff --git a/detectmatelibrary_tests/test_common/test_config_roundtrip.py b/tests/test_common/test_config_roundtrip.py similarity index 99% rename from detectmatelibrary_tests/test_common/test_config_roundtrip.py rename to tests/test_common/test_config_roundtrip.py index 6659a9e1..24fee611 100644 --- a/detectmatelibrary_tests/test_common/test_config_roundtrip.py +++ b/tests/test_common/test_config_roundtrip.py @@ -25,7 +25,7 @@ class MockupDetectorConfig(BasicConfig): def load_test_config() -> dict: - with open("detectmatelibrary_tests/test_folder/test_config.yaml", 'r') as file: + with open("tests/test_folder/test_config.yaml", 'r') as file: return yaml.safe_load(file) diff --git a/detectmatelibrary_tests/test_common/test_core.py b/tests/test_common/test_core.py similarity index 100% rename from detectmatelibrary_tests/test_common/test_core.py rename to tests/test_common/test_core.py diff --git a/detectmatelibrary_tests/test_common/test_core_detector.py b/tests/test_common/test_core_detector.py similarity index 100% rename from detectmatelibrary_tests/test_common/test_core_detector.py rename to tests/test_common/test_core_detector.py diff --git a/detectmatelibrary_tests/test_common/test_core_parser.py b/tests/test_common/test_core_parser.py similarity index 100% rename from detectmatelibrary_tests/test_common/test_core_parser.py rename to tests/test_common/test_core_parser.py diff --git a/detectmatelibrary_tests/test_common/test_extract_timestamp.py b/tests/test_common/test_extract_timestamp.py similarity index 100% rename from detectmatelibrary_tests/test_common/test_extract_timestamp.py rename to tests/test_common/test_extract_timestamp.py diff --git a/detectmatelibrary_tests/test_common/test_fit_logic.py b/tests/test_common/test_fit_logic.py similarity index 100% rename from detectmatelibrary_tests/test_common/test_fit_logic.py rename to tests/test_common/test_fit_logic.py diff --git a/detectmatelibrary_tests/test_common/test_persist_config.py b/tests/test_common/test_persist_config.py similarity index 100% rename from detectmatelibrary_tests/test_common/test_persist_config.py rename to tests/test_common/test_persist_config.py diff --git a/detectmatelibrary_tests/test_detectors/__init__.py b/tests/test_detectors/__init__.py similarity index 100% rename from detectmatelibrary_tests/test_detectors/__init__.py rename to tests/test_detectors/__init__.py diff --git a/detectmatelibrary_tests/test_detectors/dummy_detector.py b/tests/test_detectors/dummy_detector.py similarity index 100% rename from detectmatelibrary_tests/test_detectors/dummy_detector.py rename to tests/test_detectors/dummy_detector.py diff --git a/detectmatelibrary_tests/test_detectors/test_bigram_frequency_detector.py b/tests/test_detectors/test_bigram_frequency_detector.py similarity index 99% rename from detectmatelibrary_tests/test_detectors/test_bigram_frequency_detector.py rename to tests/test_detectors/test_bigram_frequency_detector.py index 22937cee..65bf5836 100644 --- a/detectmatelibrary_tests/test_detectors/test_bigram_frequency_detector.py +++ b/tests/test_detectors/test_bigram_frequency_detector.py @@ -18,7 +18,7 @@ from detectmatelibrary.helper.from_to import From import detectmatelibrary.schemas as schemas from detectmatelibrary.utils.aux import time_test_mode -from detectmatelibrary_tests.test_pipelines.test_configuration_engine import AUDIT_LOG +from tests.test_pipelines.test_configuration_engine import AUDIT_LOG # Set time test mode for consistent timestamps time_test_mode() @@ -226,7 +226,7 @@ def test_detect_known_value_alert(self): "remove_spaces": True, "remove_punctuation": True, "lowercase": True, - "path_templates": "detectmatelibrary_tests/test_folder/audit_templates.txt", + "path_templates": "tests/test_folder/audit_templates.txt", }, } } diff --git a/detectmatelibrary_tests/test_detectors/test_charset_detector.py b/tests/test_detectors/test_charset_detector.py similarity index 98% rename from detectmatelibrary_tests/test_detectors/test_charset_detector.py rename to tests/test_detectors/test_charset_detector.py index 8fe52531..9bef1155 100644 --- a/detectmatelibrary_tests/test_detectors/test_charset_detector.py +++ b/tests/test_detectors/test_charset_detector.py @@ -16,7 +16,7 @@ from detectmatelibrary.helper.from_to import From import detectmatelibrary.schemas as schemas from detectmatelibrary.utils.aux import time_test_mode -from detectmatelibrary_tests.test_pipelines.test_configuration_engine import AUDIT_LOG +from tests.test_pipelines.test_configuration_engine import AUDIT_LOG # Set time test mode for consistent timestamps time_test_mode() @@ -279,7 +279,7 @@ def test_detect_unknown_chars_reported_per_variable(self): "remove_spaces": True, "remove_punctuation": True, "lowercase": True, - "path_templates": "detectmatelibrary_tests/test_folder/audit_templates.txt", + "path_templates": "tests/test_folder/audit_templates.txt", }, } } diff --git a/detectmatelibrary_tests/test_detectors/test_mismatch_warnings.py b/tests/test_detectors/test_mismatch_warnings.py similarity index 100% rename from detectmatelibrary_tests/test_detectors/test_mismatch_warnings.py rename to tests/test_detectors/test_mismatch_warnings.py diff --git a/detectmatelibrary_tests/test_detectors/test_new_event_detector.py b/tests/test_detectors/test_new_event_detector.py similarity index 97% rename from detectmatelibrary_tests/test_detectors/test_new_event_detector.py rename to tests/test_detectors/test_new_event_detector.py index 1ca4065b..0f9c630d 100644 --- a/detectmatelibrary_tests/test_detectors/test_new_event_detector.py +++ b/tests/test_detectors/test_new_event_detector.py @@ -16,7 +16,7 @@ from detectmatelibrary.utils.aux import time_test_mode from detectmatelibrary.common._core_op._fit_logic import ConfigState, TrainState from detectmatelibrary.constants import GLOBAL_EVENT_ID -from detectmatelibrary_tests.test_pipelines.test_configuration_engine import AUDIT_LOG +from tests.test_pipelines.test_configuration_engine import AUDIT_LOG # Set time test mode for consistent timestamps @@ -144,7 +144,7 @@ def test_detect_known_event_id_no_alert(self): "remove_spaces": True, "remove_punctuation": True, "lowercase": True, - "path_templates": "detectmatelibrary_tests/test_folder/audit_templates.txt", + "path_templates": "tests/test_folder/audit_templates.txt", }, } } diff --git a/detectmatelibrary_tests/test_detectors/test_new_value_combo_detector.py b/tests/test_detectors/test_new_value_combo_detector.py similarity index 99% rename from detectmatelibrary_tests/test_detectors/test_new_value_combo_detector.py rename to tests/test_detectors/test_new_value_combo_detector.py index a6b93811..fdcef029 100644 --- a/detectmatelibrary_tests/test_detectors/test_new_value_combo_detector.py +++ b/tests/test_detectors/test_new_value_combo_detector.py @@ -5,7 +5,7 @@ from detectmatelibrary.helper.from_to import From import detectmatelibrary.schemas as schemas from detectmatelibrary.utils.aux import time_test_mode -from detectmatelibrary_tests.test_pipelines.test_configuration_engine import AUDIT_LOG +from tests.test_pipelines.test_configuration_engine import AUDIT_LOG # Set time test mode for consistent timestamps time_test_mode() @@ -548,7 +548,7 @@ def test_configure_only_selects_stable_event_types(self): "method_type": "matcher_parser", "auto_config": False, "log_format": "type= msg=audit(