diff --git a/detectmatelibrary_tests/test_common/test_core.py b/detectmatelibrary_tests/test_common/test_core.py index bffafcfc..114a74bc 100644 --- a/detectmatelibrary_tests/test_common/test_core.py +++ b/detectmatelibrary_tests/test_common/test_core.py @@ -1,4 +1,5 @@ -from detectmatelibrary.common._core_op._fit_logic import ConfigState, TrainState +from detectmatelibrary.common._core_op._fit_logic import FitLogicState + from detectmatelibrary.common.core import CoreConfig, CoreComponent from detectmatelibrary.common._config import BasicConfig @@ -34,6 +35,15 @@ class MockConfigWithTraining(CoreConfig): } +def _make_log(i: int) -> schemas.LogSchema: + return schemas.LogSchema({ + "__version__": "1.0.0", + "logID": str(i), + "logSource": "test", + "hostname": "test_hostname" + }) + + class MockComponent(CoreComponent): def __init__(self, name: str, config: MockConfig = MockConfig()) -> None: super().__init__(name=name, type_="Dummy", config=config) @@ -266,7 +276,7 @@ def test_training_force_stop(self) -> None: for i in range(10): if i == 2: - component.fitlogic.train_state = TrainState.STOP_TRAINING + component.update_state("stop_training") component.process( schemas.LogSchema({ "__version__": "1.0.0", @@ -280,7 +290,7 @@ def test_training_force_stop(self) -> None: def test_training_keep_training(self) -> None: component = MockComponentWithTraining(name="Dummy6") - component.fitlogic.train_state = TrainState.KEEP_TRAINING + component.update_state("keep_training") for i in range(10): component.process( @@ -294,18 +304,10 @@ def test_training_keep_training(self) -> None: assert len(component.train_data) == 10 - def _make_log(self, i: int) -> schemas.LogSchema: - return schemas.LogSchema({ - "__version__": "1.0.0", - "logID": str(i), - "logSource": "test", - "hostname": "test_hostname" - }) - def test_configuration(self) -> None: component = MockComponentWithConfigure(name="DummyCfg1") - results = [component.process(self._make_log(i)) for i in range(10)] + results = [component.process(_make_log(i)) for i in range(10)] assert component.fitlogic.data_used_configure == 3 assert len(component.configure_data) == 3 @@ -315,26 +317,31 @@ def test_configuration(self) -> None: def test_configuration_returns_none_during_configure(self) -> None: component = MockComponentWithConfigure(name="DummyCfg2") - results = [component.process(self._make_log(i)) for i in range(3)] + results = [component.process(_make_log(i)) for i in range(3)] assert all(r is None for r in results) + def test_wrong_update_state(self) -> None: + component = MockComponentWithConfigure(name="DummyCfg3") + with pytest.warns(UserWarning): + component.update_state("incorrect_state") + def test_configuration_force_stop(self) -> None: component = MockComponentWithConfigure(name="DummyCfg3") - component.fitlogic.configure_state = ConfigState.STOP_CONFIGURE + component.update_state("stop_configuring") for i in range(10): - component.process(self._make_log(i)) + component.process(_make_log(i)) assert len(component.configure_data) == 0 assert component.set_configuration_called == 0 def test_configuration_keep_configure(self) -> None: component = MockComponentWithConfigure(name="DummyCfg4") - component.fitlogic.configure_state = ConfigState.KEEP_CONFIGURE + component.update_state("keep_configuring") for i in range(10): - component.process(self._make_log(i)) + component.process(_make_log(i)) assert len(component.configure_data) == 10 assert component.set_configuration_called == 0 @@ -346,7 +353,7 @@ def test_configuration_before_training(self) -> None: component = MockComponentWithConfigureAndTraining(name="DummyCfg5", config=config) for i in range(10): - component.process(self._make_log(i)) + component.process(_make_log(i)) assert len(component.configure_data) == 2 assert len(component.train_data) == 3 @@ -359,7 +366,7 @@ def test_configuration_before_training_with_buffer(self) -> None: component = MockComponentWithConfigureAndTraining(name="DummyCfg5", config=config) for i in range(10): - component.process(self._make_log(i)) + component.process(_make_log(i)) assert len(component.configure_data) == 2 assert len(component.train_data) == 5 @@ -369,7 +376,7 @@ def test_set_configuration_called_once(self) -> None: component = MockComponentWithConfigure(name="DummyCfg6") for i in range(component.config.data_use_configure + 5): # type: ignore[operator] - component.process(self._make_log(i)) + component.process(_make_log(i)) assert component.set_configuration_called == 1 @@ -396,24 +403,16 @@ def run(self, input_, output_) -> bool: class TestPostTrain: - def _make_log(self, i: int) -> schemas.LogSchema: - return schemas.LogSchema({ - "__version__": "1.0.0", - "logID": str(i), - "logSource": "test", - "hostname": "test_hostname" - }) - def test_post_train_called_once_after_training(self) -> None: component = MockComponentWithPostTrain(name="PostTrain1") for i in range(10): - component.process(self._make_log(i)) + component.process(_make_log(i)) assert component.post_train_called == 1 def test_post_train_not_called_without_training(self) -> None: component = MockComponentWithPostTrain(name="PostTrain2", config=CoreConfig()) for i in range(10): - component.process(self._make_log(i)) + component.process(_make_log(i)) assert component.post_train_called == 0 def test_post_train_called_on_first_detection_item(self) -> None: @@ -421,15 +420,91 @@ def test_post_train_called_on_first_detection_item(self) -> None: component = MockComponentWithPostTrain(name="PostTrain3") # data_use_training=3, so 4th item triggers post_train for i in range(3): - component.process(self._make_log(i)) + component.process(_make_log(i)) assert component.post_train_called == 0 - component.process(self._make_log(3)) + component.process(_make_log(3)) assert component.post_train_called == 1 # subsequent items don't re-trigger it - component.process(self._make_log(4)) + component.process(_make_log(4)) assert component.post_train_called == 1 +class TestCaseState: + def test_normal_behaviour(self) -> None: + config = CoreConfig( + data_use_configure=2, data_use_training=3, use_config_data_as_training=True + ) + component = MockComponentWithConfigureAndTraining(name="DummyCfg5", config=config) + + component.process(_make_log(0)) + assert component.get_state() == FitLogicState.DO_CONFIG.describe() + + component.process(_make_log(1)) + component.process(_make_log(2)) + assert component.get_state() == FitLogicState.DO_TRAIN.describe() + + component.process(_make_log(1)) + component.process(_make_log(2)) + component.process(_make_log(3)) + assert component.get_state() == FitLogicState.NOTHING.describe() + + def test_nothing_going_on(self) -> None: + config = CoreConfig(use_config_data_as_training=True) + component = MockComponentWithConfigureAndTraining(name="DummyCfg5", config=config) + + component.process(_make_log(0)) + assert component.get_state() == FitLogicState.NOTHING.describe() + + def test_change_state(self) -> None: + config = CoreConfig(use_config_data_as_training=True) + component = MockComponentWithConfigureAndTraining(name="DummyCfg5", config=config) + + assert component.get_state() == FitLogicState.NOTHING.describe() + + component.update_state("keep_configuring") + component.process(_make_log(0)) + assert component.get_state() == FitLogicState.DO_CONFIG.describe() + + component.update_state("stop_configuring") + component.process(_make_log(0)) + assert component.get_state() == FitLogicState.NOTHING.describe() + + component.update_state("keep_training") + component.process(_make_log(0)) + assert component.get_state() == FitLogicState.DO_TRAIN.describe() + + component.update_state("stop_training") + component.process(_make_log(0)) + assert component.get_state() == FitLogicState.NOTHING.describe() + + def test_change_state_weird_cases(self) -> None: + config = CoreConfig(use_config_data_as_training=True) + component = MockComponentWithConfigureAndTraining(name="DummyCfg5", config=config) + + assert component.get_state() == FitLogicState.NOTHING.describe() + + component.update_state("keep_configuring") + component.process(_make_log(0)) + assert component.get_state() == FitLogicState.DO_CONFIG.describe() + + component.update_state("stop_training") + component.process(_make_log(0)) + assert component.get_state() == FitLogicState.DO_CONFIG.describe() + + component.update_state("keep_training") + component.process(_make_log(0)) + assert component.get_state() == FitLogicState.DO_CONFIG.describe() + + component.update_state("stop_configuring") + component.update_state("keep_training") + component.process(_make_log(0)) + assert component.get_state() == FitLogicState.DO_TRAIN.describe() + + component.update_state("keep_configuring") + component.process(_make_log(0)) + assert component.get_state() == FitLogicState.DO_CONFIG.describe() + + class TestCoreComponentContextManager: def test_can_be_used_as_context_manager(self): component = CoreComponent(name="test", config=CoreConfig(), args_buffer=ArgsBuffer(BufferMode.NO_BUF)) diff --git a/docs/overall_architecture.md b/docs/overall_architecture.md index a0920594..e89b9ff0 100644 --- a/docs/overall_architecture.md +++ b/docs/overall_architecture.md @@ -51,6 +51,25 @@ class Component(CoreComponent): ) -> None: """Train the component with a specific input""" + def update_state(self, state: StatesL) -> None: + """ + Update the current state by request of the user + states: + * keep_training: force to keep training + * stop_training: force to stop training + * keep_configuring: force to keep configuring + * stop_configuring: force to stop configuring + """ + + def get_state(self) -> str: + """ + Return the current state of the component + states: + * Configuring: the component is doing configurations + * Training: the component is training + * Default: the component is just processing data + """ + def process(self, data: BaseSchema | bytes) -> BaseSchema | bytes | None: """Process the data in a stream fashion (Defined in the CoreComponent)""" diff --git a/src/detectmatelibrary/common/_core_op/_fit_logic.py b/src/detectmatelibrary/common/_core_op/_fit_logic.py index 7a4aa610..62dd264b 100644 --- a/src/detectmatelibrary/common/_core_op/_fit_logic.py +++ b/src/detectmatelibrary/common/_core_op/_fit_logic.py @@ -1,5 +1,7 @@ +from typing import Literal, get_args from enum import Enum +import warnings class TrainState(Enum): @@ -28,10 +30,28 @@ def describe(self) -> str: "Force stop configuration.", "Keep configuring regardless of default behavior." ] - return descriptions[self.value] +StatesL = Literal["keep_training", "stop_training", "keep_configuring", "stop_configuring"] + +def update_state( + state: StatesL, train_state: TrainState, config_state: ConfigState +) -> tuple[TrainState, ConfigState]: + if state == "keep_training": + train_state = TrainState.KEEP_TRAINING + elif state == "stop_training": + train_state = TrainState.STOP_TRAINING + elif state == "keep_configuring": + config_state = ConfigState.KEEP_CONFIGURE + elif state == "stop_configuring": + config_state = ConfigState.STOP_CONFIGURE + else: + warnings.warn(f"State {state} unknown, use: {get_args(StatesL)}") + + return train_state, config_state + + def do_training( data_use_training: int | None, index: int, train_state: TrainState ) -> bool: @@ -59,6 +79,14 @@ class FitLogicState(Enum): DO_TRAIN = 1 NOTHING = 2 + def describe(self) -> str: + descriptions = [ + "Configuring", + "Training.", + "Default" + ] + return descriptions[self.value] + class FitLogic: def __init__( @@ -67,34 +95,36 @@ def __init__( self.train_state = TrainState.DEFAULT self.configure_state = ConfigState.DEFAULT + self.last_state = FitLogicState.NOTHING - self.data_used_train = 0 - self.data_used_configure = 0 - - self._configuration_done = False - self.config_finished = False - - self._training_done = False - self.training_finished = False + self.data_used_train, self.data_used_configure = 0, 0 + self._configuration_done, self.config_finished = False, False + self._training_done, self.training_finished = False, False self.data_use_configure = data_use_configure self.data_use_training = data_use_training + def get_last_state(self) -> str: + return self.last_state.describe() + + def update_state(self, state: StatesL) -> None: + self.train_state, self.configure_state = update_state( + state=state, train_state=self.train_state, config_state=self.configure_state + ) + def finish_config(self) -> bool: if self._configuration_done and not self.config_finished: self.config_finished = True return True - return False def finish_training(self) -> bool: if self._training_done and not self.training_finished: self.training_finished = True return True - return False - def run(self) -> FitLogicState: + def __check_state(self) -> FitLogicState: if do_configure( data_use_configure=self.data_use_configure, index=self.data_used_configure, @@ -116,4 +146,8 @@ def run(self) -> FitLogicState: elif self.data_used_train > 0 and not self._training_done: self._training_done = True - return FitLogicState.NOTHING \ No newline at end of file + return FitLogicState.NOTHING + + def run(self) -> FitLogicState: + self.last_state = self.__check_state() + return self.last_state \ No newline at end of file diff --git a/src/detectmatelibrary/common/core.py b/src/detectmatelibrary/common/core.py index 8c790e6b..4e783f26 100644 --- a/src/detectmatelibrary/common/core.py +++ b/src/detectmatelibrary/common/core.py @@ -1,4 +1,4 @@ -from detectmatelibrary.common._core_op._fit_logic import FitLogicState +from detectmatelibrary.common._core_op._fit_logic import FitLogicState, StatesL from detectmatelibrary.common._core_op._schema_pipeline import SchemaPipeline from detectmatelibrary.common._core_op._fit_logic import FitLogic @@ -128,6 +128,12 @@ def __init__( ) self.buffer_train = TrainBuffer() + def update_state(self, state: StatesL) -> None: + self.fitlogic.update_state(state) + + def get_state(self) -> str: + return self.fitlogic.get_last_state() + def process(self, data: BaseSchema | bytes) -> BaseSchema | bytes | None: is_byte, data = SchemaPipeline.preprocess(self.input_schema(), data) logger.debug(f"<<{self.name}>> received:\n{data}")