fix: skip uncorrectable RAPL counter wraps

codecarbon/core/rapl.py

@@ -1,4 +1,5 @@
 from dataclasses import dataclass, field
+from typing import Optional
 
 from codecarbon.core.units import Energy, Power, Time
 from codecarbon.external.logger import logger
@@ -16,12 +17,13 @@ class RAPLFile:
     energy_delta: Energy = field(default_factory=lambda: Energy(0))
     # Power based on reading
     power: Power = field(default_factory=lambda: Power(0))
-    # Last energy reading in kWh
-    last_energy: Energy = field(default_factory=lambda: Energy(0))
+    # Last energy reading in kWh, None if it could not be read
+    last_energy: Optional[Energy] = field(default_factory=lambda: Energy(0))
     # Max value energy can hold before it wraps
     max_energy_reading: Energy = field(default_factory=lambda: Energy(0))
 
     def __post_init__(self):
+        self._warned_uncorrectable_wrap = False
         self.last_energy = self._get_value()
         try:
             with open(self.max_path, "r") as f:
@@ -44,9 +46,12 @@ def __post_init__(self):
                 )
             self.max_energy_reading = Energy.from_ujoules(0)
 
-    def _get_value(self) -> Energy:
+    def _get_value(self) -> Optional[Energy]:
         """
-        Reads the value in the file at the path
+        Reads the value in the file at the path, or None if it cannot be read.
+
+        None (rather than a 0 reading) matters : a 0 would look like a
+        wrap-around and inject a spurious delta of up to one full counter range.
         """
         try:
             with open(self.path, "r") as f:
@@ -62,21 +67,44 @@ def _get_value(self) -> Energy:
                 )
             else:
                 logger.debug("Unable to read RAPL value from %s: %s", self.path, e)
-            return Energy.from_ujoules(0)
+            return None
 
     def start(self) -> None:
         self.last_energy = self._get_value()
 
+    def _skip_sample(self, new_last_energy: Optional[Energy]) -> None:
+        self.energy_delta = Energy(0)
+        self.power = Power(0)
+        self.last_energy = new_last_energy
+
     def delta(self, duration: Time) -> None:
         """
         Compute the energy used since last call.
         """
         new_last_energy = energy = self._get_value()
+        if energy is None or self.last_energy is None:
+            # No usable reading, now or at the previous sample : we have no
+            # baseline to compute a delta from, so re-baseline silently.
+            self._skip_sample(new_last_energy)
+            return
         if self.last_energy > energy:
             logger.debug(
                 f"In RAPLFile : Current energy value ({energy}) is lower than previous value ({self.last_energy}). Assuming wrap-around! Source file : {self.path}"
             )
             energy = energy + self.max_energy_reading
+            if self.last_energy > energy:
+                # The counter went backwards and we cannot correct it (unknown
+                # max_energy_range_uj, driver reset, suspend/resume...). Skip
+                # this sample rather than reporting a negative energy delta,
+                # which would corrupt the totals.
+                if not self._warned_uncorrectable_wrap:
+                    self._warned_uncorrectable_wrap = True
+                    logger.warning(
+                        "In RAPLFile : counter went backwards and cannot be corrected for %s; skipping this sample (warned once).",
+                        self.path,
+                    )
+                self._skip_sample(new_last_energy)
+                return
         self.power = self.power.from_energies_and_delay(
             energy, self.last_energy, duration
         )

tests/test_rapl_permissions.py

@@ -110,3 +110,130 @@ def emit(self, record):
             os.chmod(energy1, stat.S_IMODE(mode_before) or 0o644)
         except Exception:
             pass
+
+
+def test_rapl_wraparound_without_max_skips_sample(tmp_path):
+    """A wrap-around with an unknown max range must not yield a negative delta."""
+    from codecarbon.core.rapl import RAPLFile
+    from codecarbon.core.units import Time
+    from codecarbon.external.logger import logger as codecarbon_logger
+
+    energy_file = tmp_path / "energy_uj"
+    energy_file.write_text("4000000000")
+
+    log_records = []
+
+    class TestHandler(logging.Handler):
+        def emit(self, record):
+            log_records.append(record)
+
+    test_handler = TestHandler()
+    test_handler.setLevel(logging.WARNING)
+    codecarbon_logger.addHandler(test_handler)
+
+    try:
+        rapl_file = RAPLFile(
+            name="package-0",
+            path=str(energy_file),
+            max_path=str(tmp_path / "does_not_exist"),
+        )
+        rapl_file.start()
+        energy_file.write_text("10000")
+        rapl_file.delta(Time.from_seconds(10))
+
+        assert rapl_file.energy_delta.kWh == 0
+        assert rapl_file.power.W == 0
+        assert any(
+            "counter went backwards" in r.getMessage() for r in log_records
+        ), f"Expected warning, got: {[r.getMessage() for r in log_records]}"
+    finally:
+        codecarbon_logger.removeHandler(test_handler)
+
+
+def test_rapl_wraparound_with_max_is_corrected(tmp_path):
+    """A wrap-around with a known max range is still corrected."""
+    from codecarbon.core.rapl import RAPLFile
+    from codecarbon.core.units import Energy, Time
+
+    energy_file = tmp_path / "energy_uj"
+    energy_file.write_text("4000000000")
+    max_file = tmp_path / "max_energy_range_uj"
+    max_file.write_text("4294967295")
+
+    rapl_file = RAPLFile(
+        name="package-0", path=str(energy_file), max_path=str(max_file)
+    )
+    rapl_file.start()
+    energy_file.write_text("10000")
+    rapl_file.delta(Time.from_seconds(10))
+
+    expected = Energy.from_ujoules(10000 + 4294967295 - 4000000000).kWh
+    assert rapl_file.energy_delta.kWh == pytest.approx(expected)
+
+
+def test_rapl_read_error_does_not_inject_energy(tmp_path):
+    """A transient read error must not be mistaken for a wrap-around."""
+    from codecarbon.core.rapl import RAPLFile
+    from codecarbon.core.units import Energy, Time
+
+    energy_file = tmp_path / "energy_uj"
+    energy_file.write_text("4000000000")
+    max_file = tmp_path / "max_energy_range_uj"
+    max_file.write_text("4294967295")
+
+    rapl_file = RAPLFile(
+        name="package-0", path=str(energy_file), max_path=str(max_file)
+    )
+    rapl_file.start()
+
+    # Unreadable content : previously this returned 0 and looked like a wrap.
+    energy_file.write_text("")
+    rapl_file.delta(Time.from_seconds(10))
+    assert rapl_file.energy_delta.kWh == 0
+    assert rapl_file.power.W == 0
+
+    # The next sample re-baselines instead of counting the gap twice.
+    energy_file.write_text("4000010000")
+    rapl_file.delta(Time.from_seconds(10))
+    assert rapl_file.energy_delta.kWh == 0
+
+    energy_file.write_text("4000020000")
+    rapl_file.delta(Time.from_seconds(10))
+    assert rapl_file.energy_delta.kWh == pytest.approx(Energy.from_ujoules(10000).kWh)
+
+
+def test_rapl_uncorrectable_wrap_warns_once(tmp_path):
+    """The uncorrectable-wrap warning must not repeat on every cycle."""
+    from codecarbon.core.rapl import RAPLFile
+    from codecarbon.core.units import Time
+    from codecarbon.external.logger import logger as codecarbon_logger
+
+    energy_file = tmp_path / "energy_uj"
+    energy_file.write_text("4000000000")
+
+    log_records = []
+
+    class TestHandler(logging.Handler):
+        def emit(self, record):
+            log_records.append(record)
+
+    test_handler = TestHandler()
+    test_handler.setLevel(logging.WARNING)
+    codecarbon_logger.addHandler(test_handler)
+    try:
+        rapl_file = RAPLFile(
+            name="package-0",
+            path=str(energy_file),
+            max_path=str(tmp_path / "does_not_exist"),
+        )
+        rapl_file.start()
+        for value in ("10000", "9000", "8000"):
+            energy_file.write_text(value)
+            rapl_file.delta(Time.from_seconds(10))
+
+        backwards = [
+            r for r in log_records if "counter went backwards" in r.getMessage()
+        ]
+        assert len(backwards) == 1, [r.getMessage() for r in log_records]
+    finally:
+        codecarbon_logger.removeHandler(test_handler)