From b4d72b46fcac83b04da68af91fc952b857495589 Mon Sep 17 00:00:00 2001 From: Jaspal Singh Date: Tue, 22 Sep 2026 15:36:17 +0000 Subject: [PATCH] gpu memory check --- .../plugins/inband/amdsmi/amdsmi_analyzer.py | 72 +++++++++++++++++++ .../plugins/inband/amdsmi/analyzer_args.py | 16 ++++- test/unit/plugin/test_amdsmi_analyzer.py | 71 +++++++++++++++--- 3 files changed, 147 insertions(+), 12 deletions(-) diff --git a/nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py b/nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py index da427753..85985a3a 100644 --- a/nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py +++ b/nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py @@ -441,6 +441,73 @@ def check_amdsmi_metric_ecc_totals(self, amdsmi_metric_data: list[AmdSmiMetric]) console_log=True, ) + def check_gpu_memory( + self, + amdsmi_metric_data: list[AmdSmiMetric], + minimum_available_percent: float, + ) -> None: + """Check the minimum free VRAM percentage for each GPU.""" + for metric in amdsmi_metric_data: + memory = metric.mem_usage + total_vram = memory.total_vram if memory is not None else None + free_vram = memory.free_vram if memory is not None else None + values = { + "gpu": metric.gpu, + "total_vram": total_vram.value if total_vram is not None else None, + "free_vram": free_vram.value if free_vram is not None else None, + "unit": total_vram.unit if total_vram is not None else None, + "minimum_available_percent": minimum_available_percent, + } + + if total_vram is None or free_vram is None: + self._log_event( + category=EventCategory.PLATFORM, + description=f"GPU {metric.gpu} VRAM availability is not available", + priority=EventPriority.WARNING, + data=values, + console_log=True, + ) + continue + + try: + total_value = float(total_vram.value) + free_value = float(free_vram.value) + except (TypeError, ValueError): + self._log_event( + category=EventCategory.PLATFORM, + description=f"GPU {metric.gpu} VRAM availability is invalid", + priority=EventPriority.WARNING, + data=values, + console_log=True, + ) + continue + + if total_value <= 0: + self._log_event( + category=EventCategory.PLATFORM, + description=f"GPU {metric.gpu} total VRAM is invalid", + priority=EventPriority.WARNING, + data=values, + console_log=True, + ) + continue + + available_percent = free_value / total_value * 100 + if available_percent < minimum_available_percent: + self._log_event( + category=EventCategory.PLATFORM, + description=( + f"GPU {metric.gpu} free VRAM is {available_percent:.2f}% " + f"(minimum {minimum_available_percent:.2f}%)" + ), + priority=EventPriority.WARNING, + data={ + **values, + "available_percent": available_percent, + }, + console_log=True, + ) + def check_amdsmi_metric_ecc(self, amdsmi_metric_data: list[AmdSmiMetric]): """Check ECC counts in all blocks for all GPUs @@ -959,6 +1026,11 @@ def analyze_data( args.l0_to_recovery_count_error_threshold, args.l0_to_recovery_count_warning_threshold or 1, ) + if args.gpu_memory: + self.check_gpu_memory( + data.metric, + args.gpu_memory.minimum_available_percent, + ) self.check_amdsmi_metric_ecc_totals(data.metric) self.check_amdsmi_metric_ecc(data.metric) diff --git a/nodescraper/plugins/inband/amdsmi/analyzer_args.py b/nodescraper/plugins/inband/amdsmi/analyzer_args.py index cd08bd49..b3832b7d 100644 --- a/nodescraper/plugins/inband/amdsmi/analyzer_args.py +++ b/nodescraper/plugins/inband/amdsmi/analyzer_args.py @@ -26,12 +26,22 @@ from datetime import datetime from typing import Optional -from pydantic import Field +from pydantic import BaseModel, Field from nodescraper.models import AnalyzerArgs from nodescraper.plugins.inband.amdsmi.amdsmidata import AmdSmiDataModel +class GpuMemoryConfig(BaseModel): + """GPU VRAM availability threshold.""" + + minimum_available_percent: float = Field( + ge=0, + le=100, + description="Minimum free VRAM percentage required for each GPU.", + ) + + class AmdSmiAnalyzerArgs(AnalyzerArgs): check_static_data: bool = Field( default=False, @@ -63,6 +73,10 @@ class AmdSmiAnalyzerArgs(AnalyzerArgs): default=None, description="Expected firmware versions keyed by amd-smi fw_id (e.g. PLDM_BUNDLE).", ) + gpu_memory: Optional[GpuMemoryConfig] = Field( + default=None, + description="Minimum free VRAM threshold to validate for each GPU.", + ) l0_to_recovery_count_error_threshold: Optional[int] = Field( default=3, description="L0-to-recovery count above which an error is raised.", diff --git a/test/unit/plugin/test_amdsmi_analyzer.py b/test/unit/plugin/test_amdsmi_analyzer.py index 560a9ba4..a104e122 100644 --- a/test/unit/plugin/test_amdsmi_analyzer.py +++ b/test/unit/plugin/test_amdsmi_analyzer.py @@ -882,10 +882,24 @@ def _minimal_amdsmi_metric( ecc: Optional[MetricEccTotals] = None, ecc_blocks: Optional[dict] = None, power_management: Optional[str] = None, + mem_usage: Optional[dict] = None, ) -> AmdSmiMetric: """Build minimal AmdSmiMetric for PCIe/ECC tests with all required fields present.""" pcie_dict = pcie.model_dump() if pcie is not None else {k: None for k in _PCIE_KEYS} ecc_dict = ecc.model_dump() if ecc is not None else {k: None for k in _ECC_TOTALS_KEYS} + mem_usage_dict = { + "total_vram": None, + "used_vram": None, + "free_vram": None, + "total_visible_vram": None, + "used_visible_vram": None, + "free_visible_vram": None, + "total_gtt": None, + "used_gtt": None, + "free_gtt": None, + } + if mem_usage is not None: + mem_usage_dict.update(mem_usage) return AmdSmiMetric.model_validate( { "gpu": gpu, @@ -917,17 +931,7 @@ def _minimal_amdsmi_metric( "perf_level": None, "xgmi_err": None, "energy": None, - "mem_usage": { - "total_vram": None, - "used_vram": None, - "free_vram": None, - "total_visible_vram": None, - "used_visible_vram": None, - "free_visible_vram": None, - "total_gtt": None, - "used_gtt": None, - "free_gtt": None, - }, + "mem_usage": mem_usage_dict, "throttle": {}, } ) @@ -1002,10 +1006,55 @@ def test_analyze_data_expected_power_management(mock_analyzer): assert not any("power_management mismatch" in e.description for e in result.events) +def test_check_gpu_memory_meets_minimum(mock_analyzer): + """GPU VRAM availability at the configured minimum passes.""" + analyzer = mock_analyzer + metrics = [ + _minimal_amdsmi_metric( + 0, + mem_usage={ + "total_vram": {"value": 100, "unit": "B"}, + "free_vram": {"value": 95, "unit": "B"}, + }, + ) + ] + + analyzer.check_gpu_memory(metrics, 95) + + assert not analyzer.result.events + + +def test_check_gpu_memory_below_minimum_logs_warning(mock_analyzer): + """GPU VRAM availability below the configured minimum logs a warning.""" + analyzer = mock_analyzer + metrics = [ + _minimal_amdsmi_metric( + 1, + mem_usage={ + "total_vram": {"value": 100, "unit": "B"}, + "free_vram": {"value": 90, "unit": "B"}, + }, + ) + ] + + analyzer.check_gpu_memory(metrics, 95) + + assert len(analyzer.result.events) == 1 + event = analyzer.result.events[0] + assert event.priority == EventPriority.WARNING + assert "GPU 1 free VRAM is 90.00%" in event.description + assert event.data["available_percent"] == 90.0 + assert event.data["minimum_available_percent"] == 95 + + def test_amdsmi_analyzer_args_rejects_unknown_fields(): """Plugin config must only use declared AmdSmiAnalyzerArgs fields.""" from pydantic import ValidationError + args = AmdSmiAnalyzerArgs.model_validate({"gpu_memory": {"minimum_available_percent": 95}}) + assert args.gpu_memory is not None + assert args.gpu_memory.minimum_available_percent == 95 + with pytest.raises(ValidationError): AmdSmiAnalyzerArgs.model_validate( {"expected_power_management": "DISABLED", "not_a_field": 1}