use optional dependencies

.github/workflows/python-app.yml

@@ -30,7 +30,7 @@ jobs:
         run: uv run --dev prek run -a
 
       - name: Test with pytest
-        run: uv run --dev pytest -s
+        run: uv run --dev --extra dataframes --extra llm pytest -s
 
       # integration tests for DetectMateService
       - name: Checkout DetectMateService

README.md

@@ -43,13 +43,13 @@ protoc --proto_path=src/detectmatelibrary/schemas/ --python_out=src/detectmateli
 Run the tests:
 
 ```bash
-uv run pytest -q
+uv run --dev pytest -q
 ```
 
 Run the tests with coverage (add --cov-report=html to generate an HTML report):
 
 ```bash
-uv run pytest --cov=. --cov-report=term-missing
+uv run --dev pytest --cov=. --cov-report=term-missing
 ```
 
 ## Workspace generator (`mate create`)

docs/development.md

@@ -36,7 +36,7 @@ uv run prek run -a
 
 ## Add tests and run pytest
 
-In oder to run the tests run the following command:
+In order to run the tests run the following command. The `dev` group already includes the `full` extra, so all optional dependencies are installed automatically:
 
 ```bash
 uv run --dev pytest

docs/installation.md

@@ -22,13 +22,53 @@ To install it in a different venv as a library:
 uv pip install --no-cache-dir <directory_detectmatelibrary>
 ```
 
+### Optional dependencies
+
+Not all features require the same dependencies. DetectMateLibrary uses optional extras so you only install what you need:
+Optional Dependency Groups https://pydevtools.com/handbook/explanation/what-are-optional-dependencies-and-dependency-groups/
+
+| Extra | Installs | When you need it |
+|---|---|---|
+| `llm` | `openai`, `tenacity`, `scipy`, `scikit-learn`, `tiktoken`, `pandas` | Using the `LogBatcherParser` (LLM-based log parsing) |
+| `dataframes` | `pandas`, `polars` | Using `EventDataFrame`, `ChunkedEventDataFrame`, or `DataNormalizer` |
+| `polars-rtcompat` | `polars[rtcompat]` | Running on older CPUs without AVX2 support (e.g. some VMs or embedded hardware); not needed for standard deployments |
+| `full` | `llm` + `dataframes` + `polars-rtcompat` | Installing every optional extra at once |
+
+Install an extra with `uv sync`:
+
+```bash
+uv sync --extra dataframes
+```
+
+Or with `uv pip install` / pip when installing as a library:
+
+```bash
+uv pip install "detectmatelibrary[dataframes]"
+# or
+pip install "detectmatelibrary[dataframes]"
+```
+
+Combine multiple extras if needed:
+
+```bash
+uv sync --extra dataframes --extra polars-rtcompat
+```
+
+Or install everything at once with the `full` extra:
+
+```bash
+uv sync --extra full
+# or
+uv pip install "detectmatelibrary[full]"
+```
+
 ## Developer setup
 
 **Purpose**: prepare a development environment with test and lint tooling.
 
 ### Step 1: Install Python development dependencies & pre-commit hooks
 
-- Install dev dependencies (testing, linters, formatters):
+- Install dev dependencies (testing, linters, formatters). The `dev` group also pulls in the `full` extra, so every optional dependency (LLM, dataframes, polars-rtcompat) is installed too:
 
 ```bash
 uv sync --dev
@@ -70,7 +110,7 @@ protoc \
 
 ### Step 3: Run unit tests
 
-Run the full test suite:
+The full test suite covers dataframe and LLM-parser code, so all extras must be present. Since `uv sync --dev` already installs the `full` extra, run all tests with:
 
 ```bash
 uv run --dev pytest -s

docs/parsers/logbatcher_parser.md

@@ -7,6 +7,16 @@ LLM-based log parser that infers event templates from raw log messages using any
 | **Input**  | [LogSchema](../schemas.md)    | Raw log string                           |
 | **Output** | [ParserSchema](../schemas.md) | Structured log with template and variables |
 
+## Installation
+
+`LogBatcherParser` requires the `llm` optional extra:
+
+```bash
+pip install "detectmatelibrary[llm]"
+# or with uv
+uv sync --extra llm
+```
+
 ## Overview
 
 `LogBatcherParser` wraps the [LogBatcher](https://github.com/LogIntelligence/LogBatcher) engine (MIT, LogIntelligence 2024) as a `CoreParser`. Parsing proceeds in two phases:

pyproject.toml

@@ -12,14 +12,7 @@ dependencies = [
     "pydantic>=2.13.3",
     "pyyaml>=6.0.3",
     "regex>=2025.11.3",
-    "openai>=2.26.0",
-    "tenacity>=9.1.4",
-    "scipy>=1.17.1",
-    "scikit-learn>=1.8.0",
-    "tiktoken>=0.12.0",
     "numpy>=2.3.2",
-    "pandas>=2.3.2",
-    "polars>=1.40.1",
     "detectmateperformance>=0.1.0",
     "msgpack>=1.0.0",
     "fsspec>=2024.1.0",
@@ -33,12 +26,31 @@ dev = [
     "prek>=0.3.10",
     "pytest>=8.4.2",
     "pytest-cov>=6.2.1",
+    "detectmatelibrary[full]",
 ]
 
 [project.optional-dependencies]
+llm = [
+    "openai>=2.26.0",
+    "tenacity>=9.1.4",
+    "scipy>=1.17.1",
+    "scikit-learn>=1.8.0",
+    "tiktoken>=0.12.0",
+    "pandas>=2.3.2",
+]
+dataframes = [
+    "pandas>=2.3.2",
+    "polars>=1.40.1",
+]
 polars-rtcompat = [
     "polars[rtcompat]>=1.38.1",
 ]
+# install all optional extras at once:  uv pip install detectmatelibrary[full]
+full = [
+    "detectmatelibrary[llm]",
+    "detectmatelibrary[dataframes]",
+    "detectmatelibrary[polars-rtcompat]",
+]
 
 [tool.setuptools.dynamic]
 version = {attr = "detectmatelibrary.metadata.__version__"}

src/detectmatelibrary/utils/persistency/__init__.py

@@ -1,8 +1,8 @@
+from typing import Any
+
 from .event_persistency import EventPersistency
 from .persistency_saver import PersistencySaver, PersistencySaverConfig, PersistencyLoadError, save, load
 from .event_data_structures.base import EventDataStructure
-from .event_data_structures.dataframes.event_dataframe import EventDataFrame
-from .event_data_structures.dataframes.chunked_event_dataframe import ChunkedEventDataFrame
 from .event_data_structures.trackers.stability.stability_tracker import EventStabilityTracker
 
 __all__ = [
@@ -17,3 +17,20 @@
     "save",
     "load",
 ]
+
+_DATAFRAME_EXPORTS = {"EventDataFrame", "ChunkedEventDataFrame"}
+
+
+def __getattr__(name: str) -> Any:
+    if name in _DATAFRAME_EXPORTS:
+        try:
+            from .event_data_structures.dataframes import EventDataFrame, ChunkedEventDataFrame
+        except ImportError as e:
+            raise ImportError(
+                f"'{name}' requires the 'dataframes' extra: "
+                "pip install 'detectmatelibrary[dataframes]'"
+            ) from e
+        globals()["EventDataFrame"] = EventDataFrame
+        globals()["ChunkedEventDataFrame"] = ChunkedEventDataFrame
+        return globals()[name]
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

src/detectmatelibrary/utils/persistency/persistency_saver.py

@@ -12,10 +12,6 @@
 import fsspec
 
 from detectmatelibrary.utils.persistency.event_data_structures.base import EventDataStructure
-from detectmatelibrary.utils.persistency.event_data_structures.dataframes import (
-    EventDataFrame,
-    ChunkedEventDataFrame,
-)
 from detectmatelibrary.utils.persistency.event_data_structures.trackers import (
     EventTracker,
     EventStabilityTracker,
@@ -26,10 +22,33 @@
 _BACKEND_REGISTRY: dict[str, type[EventDataStructure]] = {
     "EventTracker": EventTracker,
     "EventStabilityTracker": EventStabilityTracker,
-    "EventDataFrame": EventDataFrame,
-    "ChunkedEventDataFrame": ChunkedEventDataFrame,
 }
 
+_DATAFRAME_BACKENDS = {"EventDataFrame", "ChunkedEventDataFrame"}
+
+
+def _get_backend_cls(name: str) -> type[EventDataStructure]:
+    if name in _BACKEND_REGISTRY:
+        return _BACKEND_REGISTRY[name]
+    if name in _DATAFRAME_BACKENDS:
+        try:
+            from detectmatelibrary.utils.persistency.event_data_structures.dataframes import (
+                ChunkedEventDataFrame,
+                EventDataFrame,
+            )
+        except ImportError as e:
+            raise PersistencyLoadError(
+                f"Backend '{name}' requires the 'dataframes' extra: "
+                "pip install 'detectmatelibrary[dataframes]'"
+            ) from e
+        df_registry: dict[str, type[EventDataStructure]] = {
+            "EventDataFrame": EventDataFrame,
+            "ChunkedEventDataFrame": ChunkedEventDataFrame,
+        }
+        return df_registry[name]
+    raise PersistencyLoadError(f"Unknown backend '{name}' — cannot restore event")
+
+
 _EXTENSION_MAP: dict[str, str] = {
     "EventTracker": "msgpack",
     "EventStabilityTracker": "msgpack",
@@ -133,16 +152,12 @@ def _load(ep: EventPersistency, fs: Any, root: str) -> None:
             file_path = f"{root}/events/{event_id_str}.{ext}"
             with fs.open(file_path, "rb") as f:
                 data = f.read()
-            if backend_name not in _BACKEND_REGISTRY:
-                raise PersistencyLoadError(
-                    f"Unknown backend '{backend_name}' — cannot restore event '{event_id}'"
-                )
-            backend_cls = _BACKEND_REGISTRY[backend_name]
+            backend_cls = _get_backend_cls(backend_name)
             ep.events_data[event_id] = backend_cls.load(data, **global_kwargs)
 
         class_name = metadata.get("event_data_class")
-        if class_name and class_name in _BACKEND_REGISTRY:
-            ep.event_data_class = _BACKEND_REGISTRY[class_name]
+        if class_name and (class_name in _BACKEND_REGISTRY or class_name in _DATAFRAME_BACKENDS):
+            ep.event_data_class = _get_backend_cls(class_name)
     except PersistencyLoadError:
         raise
     except Exception as e: