From d5be98be91e4ab15f5098496c5c754f401d4afb0 Mon Sep 17 00:00:00 2001 From: David Berenstein Date: Wed, 12 Aug 2026 17:16:49 +0200 Subject: [PATCH 1/5] perf(deps): drop pandas and numpy from the default install MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The measurement core only ever used pandas for `read_csv` plus equality filters and one mean, and numpy for four `mean`/`sum` calls. Between them they account for 61 MB of a 130 MB default install and 26% of cold import time, for work the stdlib `csv`, `statistics` and `math` modules do. - parse the bundled reference CSVs with `csv.DictReader` (utf-8-sig, since impact.csv carries a BOM), coercing `impact`/`offsetRatio` to float at the boundary and mapping empty fields to None - add `DataSource.find_cloud_region()` so the provider/region filter is written once instead of six times - rewrite `FileOutput` and `IntelPowerGadget.get_cpu_details` on stdlib csv - replace numpy in `powermetrics` with `statistics.fmean` / `math.fsum` - import `prometheus_client` only when Prometheus output is requested - move pandas to the `carbonboard` extra and add an `all` meta-extra; `codecarbon/viz/` builds its DataFrame from the returned rows Default install: 130 MB -> 68 MB, 39 -> 37 packages, cold `import codecarbon` 78.4 ms -> 37.8 ms (median of 7). Behaviour changes, both fixes: `get_cloud_geo_region` returned pandas' NaN for the 30 of 40 cloud regions that have a city but no state, and now returns the city; float means are now exactly rounded rather than matching one library's summation order. `DataSource.get_cloud_emissions_data()` and `get_cpu_power_data()` return `list[dict]` instead of `DataFrame` — undocumented internal surface, but worth a release note. Co-Authored-By: Claude Opus 5 (1M context) --- codecarbon/core/cpu.py | 55 ++++++++++++++++++--------- codecarbon/core/emissions.py | 44 +++++++++------------- codecarbon/core/powermetrics.py | 14 +++---- codecarbon/emissions_tracker.py | 13 ++----- codecarbon/input.py | 59 +++++++++++++++++++++++------ codecarbon/output_methods/file.py | 62 +++++++++++++++++-------------- codecarbon/viz/data.py | 4 +- pyproject.toml | 13 ++++++- tests/test_cpu.py | 7 +++- tests/test_package_integrity.py | 40 +++++++++++++++----- tests/test_powermetrics.py | 6 ++- uv.lock | 25 ++++++++++--- 12 files changed, 223 insertions(+), 119 deletions(-) diff --git a/codecarbon/core/cpu.py b/codecarbon/core/cpu.py index e21c39fd8..adf38f9df 100644 --- a/codecarbon/core/cpu.py +++ b/codecarbon/core/cpu.py @@ -6,13 +6,15 @@ from __future__ import annotations +import csv import os import re import shutil +import statistics import subprocess import sys from functools import lru_cache -from typing import TYPE_CHECKING, Dict, Optional, Tuple +from typing import Dict, Optional, Tuple import psutil from rapidfuzz import fuzz, process, utils @@ -22,9 +24,6 @@ from codecarbon.core.util import count_cpus, detect_cpu_model from codecarbon.external.logger import logger -if TYPE_CHECKING: - import pandas as pd - # default W value per core for a CPU if no model is found in the ref csv DEFAULT_POWER_PER_CORE = 4 @@ -375,16 +374,36 @@ def get_cpu_details(self) -> Dict: self._log_values() cpu_details = {} try: - import pandas as pd - - cpu_data = pd.read_csv(self._log_file_path).dropna() - for col_name in cpu_data.columns: + with open(self._log_file_path, newline="", encoding="utf-8-sig") as f: + reader = csv.DictReader(f) + columns = reader.fieldnames or [] + # pandas named blank headers "Unnamed: "; keep those keys + # stable for anyone consuming the returned details dict. + renames = {c: f"Unnamed: {i}" for i, c in enumerate(columns) if not c} + # Intel Power Gadget appends a ragged summary block after the + # samples; ``.dropna()`` used to discard those rows, and dropping + # rows with any missing/blank field does the same thing. + rows = [ + row + for row in reader + if all(v not in (None, "") for v in row.values()) + and None not in row + ] + + for col_name in columns: if col_name in ["System Time", "Elapsed Time (sec)", "RDTSC"]: continue + try: + values = [float(row[col_name]) for row in rows] + except (TypeError, ValueError): + continue # non-numeric column, nothing to average + if not values: + continue + key = renames.get(col_name, col_name) if "Cumulative" in col_name: - cpu_details[col_name] = cpu_data[col_name].iloc[-1] + cpu_details[key] = values[-1] else: - cpu_details[col_name] = cpu_data[col_name].mean() + cpu_details[key] = statistics.fmean(values) except Exception as e: logger.info( f"Unable to read Intel Power Gadget logged file at {self._log_file_path}\n \ @@ -898,9 +917,9 @@ def __init__(self): self.model, self.tdp = self._main() @staticmethod - def _get_cpu_constant_power(match: str, cpu_power_df: pd.DataFrame) -> int: + def _get_cpu_constant_power(match: str, cpu_power_rows: list[dict]) -> int: """Extract constant power from matched CPU""" - return float(cpu_power_df[cpu_power_df["Name"] == match]["TDP"].values[0]) + return float(next(r for r in cpu_power_rows if r["Name"] == match)["TDP"]) def _get_cpu_power_from_registry(self, cpu_model_raw: str) -> Optional[int]: from codecarbon.input import DataSource @@ -913,7 +932,7 @@ def _get_cpu_power_from_registry(self, cpu_model_raw: str) -> Optional[int]: return None def _get_matching_cpu( - self, model_raw: str, cpu_df: pd.DataFrame, greedy=False + self, model_raw: str, cpu_df: list[dict], greedy=False ) -> str: """ Get matching cpu name @@ -921,7 +940,7 @@ def _get_matching_cpu( :args: model_raw (str): raw name of the cpu model detected on the machine - cpu_df (DataFrame): table containing cpu models along their tdp + cpu_df (list[dict]): rows of cpu models along their tdp greedy (default False): if multiple cpu models match with an equal ratio of similarity, greedy (True) selects the first model, @@ -945,9 +964,11 @@ def _get_matching_cpu( THRESHOLD_DIRECT: int = 100 THRESHOLD_TOKEN_SET: int = 100 + names = [row["Name"] for row in cpu_df] + direct_match = process.extractOne( model_raw, - cpu_df["Name"], + names, processor=lambda s: s.lower(), scorer=fuzz.ratio, score_cutoff=THRESHOLD_DIRECT, @@ -964,7 +985,7 @@ def _get_matching_cpu( model_raw = re.sub(r" @\s*\d+\.\d+GHz", "", model_raw) direct_match = process.extractOne( model_raw, - cpu_df["Name"], + names, processor=lambda s: s.lower(), scorer=fuzz.ratio, score_cutoff=THRESHOLD_DIRECT, @@ -974,7 +995,7 @@ def _get_matching_cpu( return direct_match[0] indirect_matches = process.extract( model_raw, - cpu_df["Name"], + names, processor=utils.default_process, scorer=fuzz.token_set_ratio, score_cutoff=THRESHOLD_TOKEN_SET, diff --git a/codecarbon/core/emissions.py b/codecarbon/core/emissions.py index 3b2f10fad..05140fae3 100644 --- a/codecarbon/core/emissions.py +++ b/codecarbon/core/emissions.py @@ -6,7 +6,7 @@ https://github.com/responsibleproblemsolving/energy-usage """ -from typing import TYPE_CHECKING, Dict, Optional +from typing import Dict, Optional from codecarbon.core import electricitymaps_api from codecarbon.core.units import EmissionsPerKWh, Energy @@ -14,9 +14,6 @@ from codecarbon.external.logger import logger from codecarbon.input import DataSource, DataSourceException -if TYPE_CHECKING: - import pandas as pd - _NORDIC_REGIONS_BY_COUNTRY = { "SWE": {"SE1", "SE2", "SE3", "SE4"}, "NOR": {"NO1", "NO2", "NO3", "NO4", "NO5"}, @@ -65,12 +62,12 @@ def get_cloud_emissions( ) return energy.kWh * (self._force_carbon_intensity_g_co2e_kwh / 1000.0) - df: pd.DataFrame = self._data_source.get_cloud_emissions_data() try: + row = self._data_source.find_cloud_region(cloud.provider, cloud.region) + if row is None: + raise KeyError(f"{cloud.provider}/{cloud.region}") emissions_per_kWh: EmissionsPerKWh = EmissionsPerKWh.from_g_per_kWh( - df.loc[ - (df["provider"] == cloud.provider) & (df["region"] == cloud.region) - ]["impact"].item() + row["impact"] ) emissions = emissions_per_kWh.kgs_per_kWh * energy.kWh # kgs except Exception as e: @@ -100,51 +97,44 @@ def get_cloud_country_name(self, cloud: CloudMetadata) -> str: """ Returns the Country Name where the cloud region is located """ - df: pd.DataFrame = self._data_source.get_cloud_emissions_data() - flags = (df["provider"] == cloud.provider) & (df["region"] == cloud.region) - selected = df.loc[flags] - if not len(selected): + row = self._data_source.find_cloud_region(cloud.provider, cloud.region) + if row is None: raise ValueError( "Unable to find country name for " f"cloud_provider={cloud.provider}, " f"cloud_region={cloud.region}" ) - return selected["country_name"].item() + return row["country_name"] def get_cloud_country_iso_code(self, cloud: CloudMetadata) -> str: """ Returns the Country ISO Code where the cloud region is located """ - df: pd.DataFrame = self._data_source.get_cloud_emissions_data() - flags = (df["provider"] == cloud.provider) & (df["region"] == cloud.region) - selected = df.loc[flags] - if not len(selected): + row = self._data_source.find_cloud_region(cloud.provider, cloud.region) + if row is None: raise ValueError( "Unable to find country ISO Code for " f"cloud_provider={cloud.provider}, " f"cloud_region={cloud.region}" ) - return selected["countryIsoCode"].item() + return row["countryIsoCode"] def get_cloud_geo_region(self, cloud: CloudMetadata) -> str: """ Returns the State/City where the cloud region is located """ - df: pd.DataFrame = self._data_source.get_cloud_emissions_data() - flags = (df["provider"] == cloud.provider) & (df["region"] == cloud.region) - selected = df.loc[flags] - if not len(selected): + row = self._data_source.find_cloud_region(cloud.provider, cloud.region) + if row is None: raise ValueError( "Unable to find State/City name for " f"cloud_provider={cloud.provider}, " f"cloud_region={cloud.region}" ) - state = selected["state"].item() - if state is not None: - return state - city = selected["city"].item() - return city + # Empty ``state`` used to arrive here as pandas' NaN, which is not None, + # so this returned NaN for the 30 of 40 rows that only have a city. + # It now falls through to the city as originally intended. + return row["state"] if row["state"] is not None else row["city"] def get_private_infra_emissions(self, energy: Energy, geo: GeoMetadata) -> float: """ diff --git a/codecarbon/core/powermetrics.py b/codecarbon/core/powermetrics.py index 62cea9e75..59befbed0 100644 --- a/codecarbon/core/powermetrics.py +++ b/codecarbon/core/powermetrics.py @@ -1,14 +1,14 @@ +import math import os import re import shutil +import statistics import subprocess import sys import time from functools import lru_cache from typing import Dict -import numpy as np - from codecarbon.core.util import detect_cpu_model from codecarbon.external.logger import logger @@ -178,8 +178,8 @@ def get_details(self) -> Dict: for chip_part in ("CPU", "GPU"): power_list = re.findall(rf"{chip_part} Power: (\d+) mW", logfile) if not power_list: - # np.mean([]) is NaN, and NaN poisons every downstream total, - # so report 0 W instead and make the situation visible. + # An empty mean is NaN, and NaN poisons every downstream + # total, so report 0 W instead and make the situation visible. logger.warning( f"Powermetrics returned no '{chip_part} Power' sample in " + f"{self._log_file_path}, reporting 0 W." @@ -188,9 +188,9 @@ def get_details(self) -> Dict: details[f"{chip_part} Energy Delta"] = 0.0 continue watts = [float(power) / 1000 for power in power_list] - details[f"{chip_part} Power"] = np.mean(watts) - details[f"{chip_part} Energy Delta"] = np.sum( - [(self._interval / 1000) * watt for watt in watts] + details[f"{chip_part} Power"] = statistics.fmean(watts) + details[f"{chip_part} Energy Delta"] = math.fsum( + (self._interval / 1000) * watt for watt in watts ) except Exception as e: logger.info( diff --git a/codecarbon/emissions_tracker.py b/codecarbon/emissions_tracker.py index 96ed00c91..3107792b2 100644 --- a/codecarbon/emissions_tracker.py +++ b/codecarbon/emissions_tracker.py @@ -619,7 +619,6 @@ def _init_output_methods(self, *, api_key: str = None): from codecarbon.output_methods.file import FileOutput from codecarbon.output_methods.http import CodeCarbonAPIOutput, HTTPOutput from codecarbon.output_methods.metrics.logfire import LogfireOutput - from codecarbon.output_methods.metrics.prometheus import PrometheusOutput methods = set(self._output_methods) if self._output_methods else set() @@ -651,6 +650,8 @@ def _init_output_methods(self, *, api_key: str = None): self.run_id = uuid.uuid4() if OutputMethod.PROMETHEUS in methods: + from codecarbon.output_methods.metrics.prometheus import PrometheusOutput + self._output_handlers.append( PrometheusOutput( self._prometheus_url, @@ -1381,15 +1382,9 @@ def _resolve_offline_country_name(self) -> None: def _validate_offline_cloud_provider(self) -> None: if not self._cloud_provider: return - df = DataSource().get_cloud_emissions_data() if ( - len( - df.loc[ - (df["provider"] == self._cloud_provider) - & (df["region"] == self._cloud_region) - ] - ) - == 0 + DataSource().find_cloud_region(self._cloud_provider, self._cloud_region) + is None ): logger.error( "Cloud Provider/Region " diff --git a/codecarbon/input.py b/codecarbon/input.py index 4ed23db2b..b4f0940bd 100644 --- a/codecarbon/input.py +++ b/codecarbon/input.py @@ -8,14 +8,12 @@ from __future__ import annotations import atexit +import csv import json from contextlib import ExitStack from importlib.resources import as_file as importlib_resources_as_file from importlib.resources import files as importlib_resources_files -from typing import TYPE_CHECKING, Any, Dict - -if TYPE_CHECKING: - import pandas as pd +from typing import Any, Dict _CACHE: Dict[str, Any] = {} _MODULE_NAME = "codecarbon" @@ -30,6 +28,27 @@ def _get_resource_path(filepath: str): return path +def _read_csv(path, numeric_columns=()) -> list[dict[str, Any]]: + """ + Read a bundled reference CSV into a list of row dicts. + + ``utf-8-sig`` is required: ``data/cloud/impact.csv`` starts with a UTF-8 BOM, + which would otherwise end up glued to the first column name. Empty fields + become ``None`` (not ``""``) so callers can test them with ``is None``, and + the columns listed in ``numeric_columns`` are coerced to ``float`` at this + boundary rather than being left as strings for callers to guess about. + """ + with open(path, newline="", encoding="utf-8-sig") as f: + rows = [] + for raw in csv.DictReader(f): + row: dict[str, Any] = {k: (v if v else None) for k, v in raw.items()} + for column in numeric_columns: + if row.get(column) is not None: + row[column] = float(row[column]) + rows.append(row) + return rows + + def _load_static_data() -> None: """ Load all static reference data at module import. @@ -37,8 +56,6 @@ def _load_static_data() -> None: Called once when codecarbon is imported. All data loaded here is immutable and shared across all tracker instances. """ - import pandas as pd - # Global energy mix - used for emissions calculations path = _get_resource_path("data/private_infra/global_energy_mix.json") with open(path) as f: @@ -46,7 +63,9 @@ def _load_static_data() -> None: # Cloud emissions data path = _get_resource_path("data/cloud/impact.csv") - _CACHE["cloud_emissions"] = pd.read_csv(path) + _CACHE["cloud_emissions"] = _read_csv( + path, numeric_columns=("impact", "offsetRatio") + ) # Carbon intensity per source path = _get_resource_path("data/private_infra/carbon_intensity_per_source.json") @@ -55,7 +74,10 @@ def _load_static_data() -> None: # CPU power data path = _get_resource_path("data/hardware/cpu_power.csv") - _CACHE["cpu_power"] = pd.read_csv(path) + # TDP is deliberately left as text: a handful of rows hold malformed values + # such as "27.29.5", and coercing here would fail the whole load instead of + # only the lookup for those CPUs (which is what happens today). + _CACHE["cpu_power"] = _read_csv(path) # Nordic country energy mix - used for emissions calculations path = _get_resource_path("data/private_infra/nordic_emissions.json") @@ -147,14 +169,27 @@ def get_global_energy_mix_data(self) -> Dict: _ensure_static_data_loaded() return _CACHE["global_energy_mix"] - def get_cloud_emissions_data(self) -> pd.DataFrame: + def get_cloud_emissions_data(self) -> list[dict[str, Any]]: """ - Returns Cloud Regions Impact Data. + Returns Cloud Regions Impact Data, as one dict per row. Data is loaded on first access and cached for all tracker instances. """ _ensure_static_data_loaded() return _CACHE["cloud_emissions"] + def find_cloud_region(self, provider: str, region: str) -> dict[str, Any] | None: + """ + Returns the cloud impact row for a provider/region pair, or None. + """ + return next( + ( + row + for row in self.get_cloud_emissions_data() + if row["provider"] == provider and row["region"] == region + ), + None, + ) + def get_country_emissions_data(self, country_iso_code: str) -> Dict: """ Returns Emissions Across Regions in a country. @@ -195,9 +230,9 @@ def get_carbon_intensity_per_source_data(self) -> Dict: _ensure_static_data_loaded() return _CACHE["carbon_intensity_per_source"] - def get_cpu_power_data(self) -> pd.DataFrame: + def get_cpu_power_data(self) -> list[dict[str, Any]]: """ - Returns CPU power Data. + Returns CPU power Data, as one dict per row. Data is loaded on first access and cached for all tracker instances. """ _ensure_static_data_loaded() diff --git a/codecarbon/output_methods/file.py b/codecarbon/output_methods/file.py index 6a13d5b41..85ba88018 100644 --- a/codecarbon/output_methods/file.py +++ b/codecarbon/output_methods/file.py @@ -2,14 +2,24 @@ import os from typing import List -import pandas as pd - from codecarbon.core.util import backup from codecarbon.external.logger import logger from codecarbon.output_methods.base_output import BaseOutput from codecarbon.output_methods.emissions_data import EmissionsData, TaskEmissionsData +def _as_csv_row(values: dict) -> dict: + """Render a data row as strings, with missing values as empty cells.""" + return {k: ("" if v is None else str(v)) for k, v in values.items()} + + +def _write_rows(path: str, fieldnames: list[str], rows: list[dict]) -> None: + with open(path, "w", newline="") as csv_file: + writer = csv.DictWriter(csv_file, fieldnames=fieldnames, extrasaction="ignore") + writer.writeheader() + writer.writerows(rows) + + class FileOutput(BaseOutput): """ Saves experiment artifacts to a file @@ -97,33 +107,30 @@ def out(self, total: EmissionsData, _): backup(self.save_file_path) file_exists = False - new_df = pd.DataFrame.from_records([dict(total.values)]) + new_row = _as_csv_row(dict(total.values)) if not file_exists: - new_df.to_csv(self.save_file_path, index=False) + _write_rows(self.save_file_path, list(new_row), [new_row]) elif self.on_csv_write == "append": - new_df = new_df.dropna(axis=1, how="all") - new_df.to_csv(self.save_file_path, mode="a", header=False, index=False) + with open(self.save_file_path, "a", newline="") as csv_file: + csv.DictWriter(csv_file, fieldnames=list(new_row)).writerow(new_row) else: - df = pd.read_csv(self.save_file_path) - df_run = df.loc[df.run_id == total.run_id] - if len(df_run) < 1: - df = pd.concat([df, new_df]) - elif len(df_run) > 1: + with open(self.save_file_path, newline="") as csv_file: + reader = csv.DictReader(csv_file) + fieldnames = reader.fieldnames or list(new_row) + rows = list(reader) + matching = [r for r in rows if r.get("run_id") == str(total.run_id)] + if len(matching) > 1: logger.warning( - f"CSV contains more than 1 ({len(df_run)})" + f"CSV contains more than 1 ({len(matching)})" + f" rows with current run ID ({total.run_id})." + "Appending instead of updating." ) - df = pd.concat([df, new_df]) + if len(matching) == 1: + matching[0].update(new_row) else: - update_values = {} - for col, val in dict(total.values).items(): - update_values[col] = df[col].dtype.type(val) - df.loc[df.run_id == total.run_id, update_values.keys()] = ( - update_values.values() - ) - df.to_csv(self.save_file_path, index=False) + rows.append(new_row) + _write_rows(self.save_file_path, fieldnames, rows) def task_out(self, data: List[TaskEmissionsData], experiment_name: str): """ @@ -135,11 +142,10 @@ def task_out(self, data: List[TaskEmissionsData], experiment_name: str): save_task_file_path = os.path.join( self.output_dir, "emissions_" + experiment_name + "_" + run_id + ".csv" ) - new_df = pd.DataFrame.from_records( - [dict(data_point.values) for data_point in data] - ) - # Filter out empty or all-NA columns only from new_df, to avoid warnings from Pandas - # see https://github.com/pandas-dev/pandas/issues/55928 - new_df = new_df.dropna(axis=1, how="all") - df = new_df - df.to_csv(save_task_file_path, index=False) + rows = [_as_csv_row(dict(data_point.values)) for data_point in data] + # Drop columns that are empty in every row, matching the previous + # dropna(axis=1, how="all"). + fieldnames = [ + column for column in rows[0] if any(row.get(column) != "" for row in rows) + ] + _write_rows(save_task_file_path, fieldnames, rows) diff --git a/codecarbon/viz/data.py b/codecarbon/viz/data.py index 1d8d02f09..848b14ba7 100644 --- a/codecarbon/viz/data.py +++ b/codecarbon/viz/data.py @@ -211,7 +211,9 @@ def get_cloud_emissions_barchart_data( "", pd.DataFrame(data={"region": [], "emissions": [], "country_name": []}), ) - cloud_emissions = self._data_source.get_cloud_emissions_data() + # DataSource returns plain row dicts; the dashboard is the only consumer + # that still wants a DataFrame, so build one here. + cloud_emissions = pd.DataFrame(self._data_source.get_cloud_emissions_data()) cloud_emissions = cloud_emissions[ ["provider", "providerName", "region", "impact", "country_name"] ] diff --git a/pyproject.toml b/pyproject.toml index 26a360338..98ad7bc48 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,8 +29,6 @@ dependencies = [ "authlib>=1.2.1", "joserfc>=1.0.0", "click", - "pandas>=2.3.3;python_version>='3.14'", - "pandas;python_version<'3.14'", "prometheus_client", "psutil >= 6.0.0", "py-cpuinfo", @@ -88,6 +86,7 @@ dev = [ "black", "mypy", "pytest", + "pandas", # no longer a runtime dep; tests read output CSVs with it "requests", "requests-mock", "responses", @@ -109,16 +108,26 @@ doc = [ ] [project.optional-dependencies] +# pandas is only needed by the dashboard now; the measurement core parses its +# reference CSVs with the stdlib csv module. carbonboard = [ "dash", "dash_bootstrap_components > 1.0.0", "fire", + "pandas>=2.3.3;python_version>='3.14'", + "pandas;python_version<'3.14'", ] # Backwards compatibility alias - will be removed in v4.0.0 viz-legacy = [ "dash", "dash_bootstrap_components > 1.0.0", "fire", + "pandas>=2.3.3;python_version>='3.14'", + "pandas;python_version<'3.14'", +] +# Everything, for users who do not want to think about extras. +all = [ + "codecarbon[carbonboard]", ] [project.scripts] diff --git a/tests/test_cpu.py b/tests/test_cpu.py index ae672c6fa..c576cbb96 100644 --- a/tests/test_cpu.py +++ b/tests/test_cpu.py @@ -238,7 +238,12 @@ def test_get_cpu_details(self, mock_setup, mock_log_values): cpu_details["Cumulative IA Energy_0(mWh)"] = round( cpu_details["Cumulative IA Energy_0(mWh)"], 3 ) - self.assertDictEqual(expected_cpu_details, cpu_details) + # Compared with a tolerance rather than assertDictEqual: the + # values are float means, and pinning one summation order's last + # ulp is not a property worth asserting. + self.assertEqual(sorted(expected_cpu_details), sorted(cpu_details)) + for key, expected in expected_cpu_details.items(): + self.assertAlmostEqual(expected, cpu_details[key], places=9) def test_setup_cli_uses_windows_backup_when_primary_missing(self): with ( diff --git a/tests/test_package_integrity.py b/tests/test_package_integrity.py index 3299664fc..4083618a2 100644 --- a/tests/test_package_integrity.py +++ b/tests/test_package_integrity.py @@ -3,9 +3,11 @@ This test should be run against the installed package, not the source. """ +import json +import subprocess +import sys from importlib import resources as importlib_resources -import pandas as pd import pytest from codecarbon.input import DataSource @@ -21,10 +23,9 @@ def test_critical_data_files_included(): # Test that we can actually read the cloud emissions data cloud_data = ds.get_cloud_emissions_data() - assert isinstance( - cloud_data, pd.DataFrame - ), "Cloud emissions data should be a DataFrame" - assert not cloud_data.empty, "Cloud emissions data should not be empty" + assert isinstance(cloud_data, list), "Cloud emissions data should be a list of rows" + assert cloud_data, "Cloud emissions data should not be empty" + assert "provider" in cloud_data[0], "BOM must not leak into the first column name" # Test carbon intensity data carbon_intensity_path = ds.carbon_intensity_per_source_path @@ -50,10 +51,8 @@ def test_cpu_power_data_included(): # Test that we can actually read the CPU power data cpu_power_data = ds.get_cpu_power_data() - assert isinstance( - cpu_power_data, pd.DataFrame - ), "CPU power data should be a DataFrame" - assert not cpu_power_data.empty, "CPU power data should not be empty" + assert isinstance(cpu_power_data, list), "CPU power data should be a list of rows" + assert cpu_power_data, "CPU power data should not be empty" def test_global_energy_mix_data_included(): @@ -150,3 +149,26 @@ def test_package_importability(): from codecarbon.output import EmissionsData assert EmissionsData is not None + + +def test_core_does_not_import_pandas_or_numpy(): + """ + pandas/numpy are dashboard-only dependencies and are not installed by a + default `pip install codecarbon`. Running a tracker end to end must not + reach for them, so assert on a fresh interpreter's sys.modules. + """ + script = ( + "import sys, tempfile, os, json\n" + "os.chdir(tempfile.mkdtemp())\n" + "from codecarbon import OfflineEmissionsTracker\n" + "t = OfflineEmissionsTracker(country_iso_code='FRA', allow_multiple_runs=True)\n" + "t.start(); t.stop()\n" + "print(json.dumps([m for m in ('pandas', 'numpy', 'prometheus_client')" + " if m in sys.modules]))\n" + ) + result = subprocess.run( + [sys.executable, "-c", script], capture_output=True, text=True, check=False + ) + assert result.returncode == 0, result.stderr + leaked = json.loads(result.stdout.strip().splitlines()[-1]) + assert leaked == [], f"core tracker path imported {leaked}" diff --git a/tests/test_powermetrics.py b/tests/test_powermetrics.py index cd1a6ca09..1913ec30a 100644 --- a/tests/test_powermetrics.py +++ b/tests/test_powermetrics.py @@ -71,7 +71,11 @@ def test_get_details(self, mock_setup, mock_log_values): ) cpu_details = powermetrics.get_details() - assert cpu_details == expected_details + # Tolerance rather than equality: these are float sums/means, and the + # exact last ulp depends on summation order, not on correctness. + assert sorted(cpu_details) == sorted(expected_details) + for key, expected in expected_details.items(): + assert cpu_details[key] == pytest.approx(expected) @mock.patch("codecarbon.core.powermetrics.ApplePowermetrics._log_values") @mock.patch("codecarbon.core.powermetrics.ApplePowermetrics._setup_cli") diff --git a/uv.lock b/uv.lock index f37050957..3e5cc9366 100644 --- a/uv.lock +++ b/uv.lock @@ -428,8 +428,6 @@ dependencies = [ { name = "click" }, { name = "joserfc" }, { name = "nvidia-ml-py" }, - { name = "pandas", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "pandas", version = "3.0.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "prometheus-client" }, { name = "psutil" }, { name = "py-cpuinfo" }, @@ -443,15 +441,26 @@ dependencies = [ ] [package.optional-dependencies] +all = [ + { name = "dash" }, + { name = "dash-bootstrap-components" }, + { name = "fire" }, + { name = "pandas", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "pandas", version = "3.0.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, +] carbonboard = [ { name = "dash" }, { name = "dash-bootstrap-components" }, { name = "fire" }, + { name = "pandas", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "pandas", version = "3.0.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, ] viz-legacy = [ { name = "dash" }, { name = "dash-bootstrap-components" }, { name = "fire" }, + { name = "pandas", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "pandas", version = "3.0.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, ] [package.dev-dependencies] @@ -462,6 +471,8 @@ dev = [ { name = "logfire" }, { name = "mktestdocs" }, { name = "mypy" }, + { name = "pandas", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "pandas", version = "3.0.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "pre-commit" }, { name = "pytest" }, { name = "pytest-cov" }, @@ -489,6 +500,7 @@ requires-dist = [ { name = "arrow" }, { name = "authlib", specifier = ">=1.2.1" }, { name = "click" }, + { name = "codecarbon", extras = ["carbonboard"], marker = "extra == 'all'" }, { name = "dash", marker = "extra == 'carbonboard'" }, { name = "dash", marker = "extra == 'viz-legacy'" }, { name = "dash-bootstrap-components", marker = "extra == 'carbonboard'", specifier = ">1.0.0" }, @@ -497,8 +509,10 @@ requires-dist = [ { name = "fire", marker = "extra == 'viz-legacy'" }, { name = "joserfc", specifier = ">=1.0.0" }, { name = "nvidia-ml-py" }, - { name = "pandas", marker = "python_full_version < '3.14'" }, - { name = "pandas", marker = "python_full_version >= '3.14'", specifier = ">=2.3.3" }, + { name = "pandas", marker = "python_full_version >= '3.14' and extra == 'carbonboard'", specifier = ">=2.3.3" }, + { name = "pandas", marker = "python_full_version >= '3.14' and extra == 'viz-legacy'", specifier = ">=2.3.3" }, + { name = "pandas", marker = "python_full_version < '3.14' and extra == 'carbonboard'" }, + { name = "pandas", marker = "python_full_version < '3.14' and extra == 'viz-legacy'" }, { name = "prometheus-client" }, { name = "psutil", specifier = ">=6.0.0" }, { name = "py-cpuinfo" }, @@ -510,7 +524,7 @@ requires-dist = [ { name = "rich" }, { name = "typer" }, ] -provides-extras = ["carbonboard", "viz-legacy"] +provides-extras = ["carbonboard", "viz-legacy", "all"] [package.metadata.requires-dev] dev = [ @@ -520,6 +534,7 @@ dev = [ { name = "logfire", specifier = ">=1.0.1" }, { name = "mktestdocs" }, { name = "mypy" }, + { name = "pandas" }, { name = "pre-commit" }, { name = "pytest" }, { name = "pytest-cov" }, From 7b5813073381d0c02dfc8109e5f0ac558c82775e Mon Sep 17 00:00:00 2001 From: David Berenstein Date: Wed, 12 Aug 2026 21:08:37 +0200 Subject: [PATCH 2/5] fix(csv): write NaN as an empty cell, matching the previous pandas output Also document why extrasaction="ignore" is unreachable. --- codecarbon/output_methods/file.py | 15 +++++++++++++-- tests/output_methods/test_file.py | 14 ++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/codecarbon/output_methods/file.py b/codecarbon/output_methods/file.py index 85ba88018..07af501f4 100644 --- a/codecarbon/output_methods/file.py +++ b/codecarbon/output_methods/file.py @@ -1,4 +1,5 @@ import csv +import math import os from typing import List @@ -9,12 +10,22 @@ def _as_csv_row(values: dict) -> dict: - """Render a data row as strings, with missing values as empty cells.""" - return {k: ("" if v is None else str(v)) for k, v in values.items()} + """Render a data row as strings, with missing values as empty cells. + + NaN is written as an empty cell, the way pandas' ``to_csv`` did, so that a + missing measurement does not end up as the literal string "nan". + """ + return { + k: ("" if v is None or (isinstance(v, float) and math.isnan(v)) else str(v)) + for k, v in values.items() + } def _write_rows(path: str, fieldnames: list[str], rows: list[dict]) -> None: with open(path, "w", newline="") as csv_file: + # extrasaction="ignore" is defensive only: a row can never carry a key + # outside `fieldnames`, because `out()` backs the file up and rewrites it + # from scratch whenever `has_valid_headers()` reports a mismatch. writer = csv.DictWriter(csv_file, fieldnames=fieldnames, extrasaction="ignore") writer.writeheader() writer.writerows(rows) diff --git a/tests/output_methods/test_file.py b/tests/output_methods/test_file.py index e8bccfdf0..9686d4493 100644 --- a/tests/output_methods/test_file.py +++ b/tests/output_methods/test_file.py @@ -1,3 +1,4 @@ +import csv import os import shutil import tempfile @@ -87,6 +88,19 @@ def test_has_valid_headers_different_order_success(self): self.assertTrue(file_output.has_valid_headers(self.emissions_data)) + def test_nan_is_written_as_an_empty_cell(self): + """NaN must round-trip as a missing value, not as the string "nan".""" + file_output = FileOutput("test.csv", self.temp_dir) + self.emissions_data.cpu_power = float("nan") + file_output.out(self.emissions_data, None) + + with open(file_output.save_file_path) as csv_file: + row = next(csv.DictReader(csv_file)) + self.assertEqual(row["cpu_power"], "") + self.assertTrue( + pd.isna(pd.read_csv(file_output.save_file_path)["cpu_power"][0]) + ) + def test_has_valid_headers_failure(self): file_output = FileOutput("test.csv", self.temp_dir) file_output.out(self.emissions_data, None) From 1190b9840539b5c48cd7100a7037459a8571c639 Mon Sep 17 00:00:00 2001 From: David Berenstein Date: Wed, 12 Aug 2026 23:10:26 +0200 Subject: [PATCH 3/5] fix(csv): write infinities as empty cells too Co-Authored-By: Claude Opus 5 (1M context) --- codecarbon/output_methods/file.py | 13 ++++++++++--- tests/output_methods/test_file.py | 27 +++++++++++++++------------ 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/codecarbon/output_methods/file.py b/codecarbon/output_methods/file.py index 07af501f4..ef37c9464 100644 --- a/codecarbon/output_methods/file.py +++ b/codecarbon/output_methods/file.py @@ -12,11 +12,18 @@ def _as_csv_row(values: dict) -> dict: """Render a data row as strings, with missing values as empty cells. - NaN is written as an empty cell, the way pandas' ``to_csv`` did, so that a - missing measurement does not end up as the literal string "nan". + Any non-finite float (NaN, +inf, -inf) is written as an empty cell. NaN + matches what pandas' ``to_csv`` did; infinities are a deliberate + divergence (``to_csv`` wrote the literal "inf") because an infinite + energy or emissions value is not a real measurement, and an empty cell + reads back as missing instead of poisoning downstream arithmetic. """ return { - k: ("" if v is None or (isinstance(v, float) and math.isnan(v)) else str(v)) + k: ( + "" + if v is None or (isinstance(v, float) and not math.isfinite(v)) + else str(v) + ) for k, v in values.items() } diff --git a/tests/output_methods/test_file.py b/tests/output_methods/test_file.py index 9686d4493..dfe86896a 100644 --- a/tests/output_methods/test_file.py +++ b/tests/output_methods/test_file.py @@ -88,18 +88,21 @@ def test_has_valid_headers_different_order_success(self): self.assertTrue(file_output.has_valid_headers(self.emissions_data)) - def test_nan_is_written_as_an_empty_cell(self): - """NaN must round-trip as a missing value, not as the string "nan".""" - file_output = FileOutput("test.csv", self.temp_dir) - self.emissions_data.cpu_power = float("nan") - file_output.out(self.emissions_data, None) - - with open(file_output.save_file_path) as csv_file: - row = next(csv.DictReader(csv_file)) - self.assertEqual(row["cpu_power"], "") - self.assertTrue( - pd.isna(pd.read_csv(file_output.save_file_path)["cpu_power"][0]) - ) + def test_non_finite_values_are_written_as_empty_cells(self): + """NaN and infinities must round-trip as missing, not as "nan"/"inf".""" + for value in (float("nan"), float("inf"), float("-inf")): + with self.subTest(value=value): + file_output = FileOutput("test.csv", self.temp_dir) + self.emissions_data.cpu_power = value + file_output.out(self.emissions_data, None) + + with open(file_output.save_file_path) as csv_file: + row = next(csv.DictReader(csv_file)) + self.assertEqual(row["cpu_power"], "") + self.assertTrue( + pd.isna(pd.read_csv(file_output.save_file_path)["cpu_power"][0]) + ) + os.remove(file_output.save_file_path) def test_has_valid_headers_failure(self): file_output = FileOutput("test.csv", self.temp_dir) From 731eab300c5d7938e9f3b7515a16c6c10e1205c7 Mon Sep 17 00:00:00 2001 From: David Berenstein Date: Thu, 13 Aug 2026 07:13:24 +0200 Subject: [PATCH 4/5] fix(deps,csv): declare numpy in the viz extras and append by on-disk header The viz extras install pandas but not numpy, which `codecarbon/viz/components.py` imports directly; numpy used to arrive transitively through pandas in the default install, so `pip install codecarbon[viz-legacy]` now breaks on import. The CSV append path passed the row dict's key order as DictWriter fieldnames, while `has_valid_headers()` accepts any permutation of the columns already on disk. Read the header row from the file instead, so a reordered file gets correctly aligned rows; an empty/headerless file is rewritten with a header. Co-Authored-By: Claude Opus 5 (1M context) --- codecarbon/output_methods/file.py | 14 ++++++++++++-- pyproject.toml | 2 ++ tests/output_methods/test_file.py | 16 ++++++++++++++++ uv.lock | 11 +++++++++++ 4 files changed, 41 insertions(+), 2 deletions(-) diff --git a/codecarbon/output_methods/file.py b/codecarbon/output_methods/file.py index ef37c9464..f8274a76b 100644 --- a/codecarbon/output_methods/file.py +++ b/codecarbon/output_methods/file.py @@ -130,8 +130,18 @@ def out(self, total: EmissionsData, _): if not file_exists: _write_rows(self.save_file_path, list(new_row), [new_row]) elif self.on_csv_write == "append": - with open(self.save_file_path, "a", newline="") as csv_file: - csv.DictWriter(csv_file, fieldnames=list(new_row)).writerow(new_row) + # Use the header already on disk as the column order: the row dict's + # key order is not guaranteed to match it (has_valid_headers() + # compares the two sorted), and trusting it would misalign columns. + with open(self.save_file_path, newline="") as csv_file: + fieldnames = next(csv.reader(csv_file), None) + if not fieldnames: + _write_rows(self.save_file_path, list(new_row), [new_row]) + else: + with open(self.save_file_path, "a", newline="") as csv_file: + csv.DictWriter( + csv_file, fieldnames=fieldnames, extrasaction="ignore" + ).writerow(new_row) else: with open(self.save_file_path, newline="") as csv_file: reader = csv.DictReader(csv_file) diff --git a/pyproject.toml b/pyproject.toml index 98ad7bc48..9e2067b21 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -114,6 +114,7 @@ carbonboard = [ "dash", "dash_bootstrap_components > 1.0.0", "fire", + "numpy", "pandas>=2.3.3;python_version>='3.14'", "pandas;python_version<'3.14'", ] @@ -122,6 +123,7 @@ viz-legacy = [ "dash", "dash_bootstrap_components > 1.0.0", "fire", + "numpy", "pandas>=2.3.3;python_version>='3.14'", "pandas;python_version<'3.14'", ] diff --git a/tests/output_methods/test_file.py b/tests/output_methods/test_file.py index dfe86896a..bf3c867df 100644 --- a/tests/output_methods/test_file.py +++ b/tests/output_methods/test_file.py @@ -149,6 +149,22 @@ def test_file_output_out_append_file_exists(self): df = pd.read_csv(os.path.join(self.temp_dir, "test.csv")) self.assertEqual(len(df), 2) + def test_file_output_out_append_respects_existing_column_order(self): + """Appending must follow the header on disk, not the row's key order.""" + file_output = FileOutput("test.csv", self.temp_dir, on_csv_write="append") + file_output.out(self.emissions_data, None) + + df = pd.read_csv(os.path.join(self.temp_dir, "test.csv")) + df = df[list(reversed(df.columns))] + df.to_csv(os.path.join(self.temp_dir, "test.csv"), index=False) + + file_output.out(self.emissions_data, None) + + df = pd.read_csv(os.path.join(self.temp_dir, "test.csv")) + self.assertEqual(len(df), 2) + self.assertEqual(df.iloc[1]["project_name"], self.emissions_data.project_name) + self.assertEqual(df.iloc[0].to_dict(), df.iloc[1].to_dict()) + def test_file_output_out_update_file_exists_no_matching_row(self): file_output = FileOutput("test.csv", self.temp_dir, on_csv_write="update") file_output.out(self.emissions_data, None) diff --git a/uv.lock b/uv.lock index 3e5cc9366..97a6ed2a0 100644 --- a/uv.lock +++ b/uv.lock @@ -445,6 +445,9 @@ all = [ { name = "dash" }, { name = "dash-bootstrap-components" }, { name = "fire" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "pandas", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "pandas", version = "3.0.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, ] @@ -452,6 +455,9 @@ carbonboard = [ { name = "dash" }, { name = "dash-bootstrap-components" }, { name = "fire" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "pandas", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "pandas", version = "3.0.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, ] @@ -459,6 +465,9 @@ viz-legacy = [ { name = "dash" }, { name = "dash-bootstrap-components" }, { name = "fire" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "pandas", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "pandas", version = "3.0.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, ] @@ -508,6 +517,8 @@ requires-dist = [ { name = "fire", marker = "extra == 'carbonboard'" }, { name = "fire", marker = "extra == 'viz-legacy'" }, { name = "joserfc", specifier = ">=1.0.0" }, + { name = "numpy", marker = "extra == 'carbonboard'" }, + { name = "numpy", marker = "extra == 'viz-legacy'" }, { name = "nvidia-ml-py" }, { name = "pandas", marker = "python_full_version >= '3.14' and extra == 'carbonboard'", specifier = ">=2.3.3" }, { name = "pandas", marker = "python_full_version >= '3.14' and extra == 'viz-legacy'", specifier = ">=2.3.3" }, From afdf4cc2f8afa5d59b434ae07f932fdfc0df250e Mon Sep 17 00:00:00 2001 From: David Berenstein Date: Thu, 13 Aug 2026 07:20:20 +0200 Subject: [PATCH 5/5] fix: keep the DataSource accessors returning DataFrames when pandas is installed Dropping pandas from the default install turned DataSource.get_cloud_emissions_data() and get_cpu_power_data() into list[dict] returns, which would break any external caller relying on the DataFrame they used to get. Internal callers now use new get_cloud_emissions_rows() / get_cpu_power_rows() helpers, so the default path stays pandas-free. The two public methods wrap those and lazily import pandas: a DataFrame when pandas is present, the rows otherwise (a user without pandas could not have been using the DataFrame anyway). Co-Authored-By: Claude Opus 5 (1M context) --- codecarbon/core/cpu.py | 2 +- codecarbon/input.py | 40 +++++++++++++++++++++++++++--- codecarbon/viz/data.py | 2 +- tests/test_cpu.py | 4 +-- tests/test_input.py | 44 ++++++++++++++++++++++++++++++--- tests/test_package_integrity.py | 4 +-- 6 files changed, 83 insertions(+), 13 deletions(-) diff --git a/codecarbon/core/cpu.py b/codecarbon/core/cpu.py index adf38f9df..f83a4742f 100644 --- a/codecarbon/core/cpu.py +++ b/codecarbon/core/cpu.py @@ -924,7 +924,7 @@ def _get_cpu_constant_power(match: str, cpu_power_rows: list[dict]) -> int: def _get_cpu_power_from_registry(self, cpu_model_raw: str) -> Optional[int]: from codecarbon.input import DataSource - cpu_power_df = DataSource().get_cpu_power_data() + cpu_power_df = DataSource().get_cpu_power_rows() cpu_matching = self._get_matching_cpu(cpu_model_raw, cpu_power_df) if cpu_matching: power = self._get_cpu_constant_power(cpu_matching, cpu_power_df) diff --git a/codecarbon/input.py b/codecarbon/input.py index b4f0940bd..76fbb3f03 100644 --- a/codecarbon/input.py +++ b/codecarbon/input.py @@ -49,6 +49,20 @@ def _read_csv(path, numeric_columns=()) -> list[dict[str, Any]]: return rows +def _as_dataframe(rows: list[dict[str, Any]]): + """ + Wrap rows in a DataFrame if pandas is available, else return them as-is. + + pandas is an optional dependency now, so the import has to stay inside the + function: nothing on the default install path may import it at module level. + """ + try: + import pandas as pd + except ImportError: + return rows + return pd.DataFrame(rows) + + def _load_static_data() -> None: """ Load all static reference data at module import. @@ -169,7 +183,7 @@ def get_global_energy_mix_data(self) -> Dict: _ensure_static_data_loaded() return _CACHE["global_energy_mix"] - def get_cloud_emissions_data(self) -> list[dict[str, Any]]: + def get_cloud_emissions_rows(self) -> list[dict[str, Any]]: """ Returns Cloud Regions Impact Data, as one dict per row. Data is loaded on first access and cached for all tracker instances. @@ -177,6 +191,16 @@ def get_cloud_emissions_data(self) -> list[dict[str, Any]]: _ensure_static_data_loaded() return _CACHE["cloud_emissions"] + def get_cloud_emissions_data(self): + """ + Returns Cloud Regions Impact Data. + + Kept for backwards compatibility: this used to return a DataFrame and + still does when pandas is installed. pandas is no longer a default + dependency, so without it the plain rows are returned instead. + """ + return _as_dataframe(self.get_cloud_emissions_rows()) + def find_cloud_region(self, provider: str, region: str) -> dict[str, Any] | None: """ Returns the cloud impact row for a provider/region pair, or None. @@ -184,7 +208,7 @@ def find_cloud_region(self, provider: str, region: str) -> dict[str, Any] | None return next( ( row - for row in self.get_cloud_emissions_data() + for row in self.get_cloud_emissions_rows() if row["provider"] == provider and row["region"] == region ), None, @@ -230,7 +254,7 @@ def get_carbon_intensity_per_source_data(self) -> Dict: _ensure_static_data_loaded() return _CACHE["carbon_intensity_per_source"] - def get_cpu_power_data(self) -> list[dict[str, Any]]: + def get_cpu_power_rows(self) -> list[dict[str, Any]]: """ Returns CPU power Data, as one dict per row. Data is loaded on first access and cached for all tracker instances. @@ -238,6 +262,16 @@ def get_cpu_power_data(self) -> list[dict[str, Any]]: _ensure_static_data_loaded() return _CACHE["cpu_power"] + def get_cpu_power_data(self): + """ + Returns CPU power Data. + + Kept for backwards compatibility: this used to return a DataFrame and + still does when pandas is installed. pandas is no longer a default + dependency, so without it the plain rows are returned instead. + """ + return _as_dataframe(self.get_cpu_power_rows()) + def get_nordic_country_energy_mix_data(self) -> Dict: """ Returns Nordic Country Energy Mix Data. diff --git a/codecarbon/viz/data.py b/codecarbon/viz/data.py index 848b14ba7..17fab13bf 100644 --- a/codecarbon/viz/data.py +++ b/codecarbon/viz/data.py @@ -213,7 +213,7 @@ def get_cloud_emissions_barchart_data( ) # DataSource returns plain row dicts; the dashboard is the only consumer # that still wants a DataFrame, so build one here. - cloud_emissions = pd.DataFrame(self._data_source.get_cloud_emissions_data()) + cloud_emissions = pd.DataFrame(self._data_source.get_cloud_emissions_rows()) cloud_emissions = cloud_emissions[ ["provider", "providerName", "region", "impact", "country_name"] ] diff --git a/tests/test_cpu.py b/tests/test_cpu.py index c576cbb96..9a6b63744 100644 --- a/tests/test_cpu.py +++ b/tests/test_cpu.py @@ -399,7 +399,7 @@ def test_get_cpu_power_from_registry_returns_none_without_match(self): mock.patch("codecarbon.input.DataSource") as mock_data_source, mock.patch.object(tdp, "_get_matching_cpu", return_value=None), ): - mock_data_source.return_value.get_cpu_power_data.return_value = ( + mock_data_source.return_value.get_cpu_power_rows.return_value = ( mock.sentinel.cpu_power_df ) @@ -407,7 +407,7 @@ def test_get_cpu_power_from_registry_returns_none_without_match(self): def test_get_matching_cpu(self): tdp = TDP() - cpu_data = DataSource().get_cpu_power_data() + cpu_data = DataSource().get_cpu_power_rows() # ======= WORKING AS EXPECTED ======== diff --git a/tests/test_input.py b/tests/test_input.py index 875e7e99a..3fb81fefa 100644 --- a/tests/test_input.py +++ b/tests/test_input.py @@ -6,6 +6,7 @@ """ import unittest +from unittest import mock class TestDataSourceCaching(unittest.TestCase): @@ -41,11 +42,11 @@ def test_get_global_energy_mix_returns_cached_data(self): self.assertIs(data, _CACHE["global_energy_mix"]) def test_get_cloud_emissions_returns_cached_data(self): - """Verify get_cloud_emissions_data() returns cached object.""" + """Verify get_cloud_emissions_rows() returns cached object.""" from codecarbon.input import _CACHE, DataSource ds = DataSource() - data = ds.get_cloud_emissions_data() + data = ds.get_cloud_emissions_rows() # Should return the exact same object from cache self.assertIs(data, _CACHE["cloud_emissions"]) @@ -61,11 +62,11 @@ def test_get_carbon_intensity_returns_cached_data(self): self.assertIs(data, _CACHE["carbon_intensity_per_source"]) def test_get_cpu_power_returns_cached_data(self): - """Verify get_cpu_power_data() returns cached object.""" + """Verify get_cpu_power_rows() returns cached object.""" from codecarbon.input import _CACHE, DataSource ds = DataSource() - data = ds.get_cpu_power_data() + data = ds.get_cpu_power_rows() # Should return the exact same object from cache self.assertIs(data, _CACHE["cpu_power"]) @@ -97,5 +98,40 @@ def test_multiple_datasource_instances_share_cache(self): self.assertIs(data1, data2) +class TestDataFrameBackwardsCompatibility(unittest.TestCase): + """The public accessors kept their pre-slimming DataFrame return type.""" + + def test_public_accessors_return_dataframes_when_pandas_is_installed(self): + pd = __import__("pandas") + from codecarbon.input import DataSource + + ds = DataSource() + for rows, frame in ( + (ds.get_cloud_emissions_rows(), ds.get_cloud_emissions_data()), + (ds.get_cpu_power_rows(), ds.get_cpu_power_data()), + ): + self.assertIsInstance(rows, list) + self.assertIsInstance(rows[0], dict) + self.assertIsInstance(frame, pd.DataFrame) + self.assertEqual(len(frame), len(rows)) + + def test_public_accessors_return_rows_without_pandas(self): + import builtins + + from codecarbon.input import DataSource + + real_import = builtins.__import__ + + def no_pandas(name, *args, **kwargs): + if name == "pandas": + raise ImportError("No module named 'pandas'") + return real_import(name, *args, **kwargs) + + ds = DataSource() + with mock.patch.object(builtins, "__import__", no_pandas): + self.assertIs(ds.get_cloud_emissions_data(), ds.get_cloud_emissions_rows()) + self.assertIs(ds.get_cpu_power_data(), ds.get_cpu_power_rows()) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_package_integrity.py b/tests/test_package_integrity.py index 4083618a2..a1bfd7c28 100644 --- a/tests/test_package_integrity.py +++ b/tests/test_package_integrity.py @@ -22,7 +22,7 @@ def test_critical_data_files_included(): assert cloud_path.exists(), f"Cloud emissions file missing: {cloud_path}" # Test that we can actually read the cloud emissions data - cloud_data = ds.get_cloud_emissions_data() + cloud_data = ds.get_cloud_emissions_rows() assert isinstance(cloud_data, list), "Cloud emissions data should be a list of rows" assert cloud_data, "Cloud emissions data should not be empty" assert "provider" in cloud_data[0], "BOM must not leak into the first column name" @@ -50,7 +50,7 @@ def test_cpu_power_data_included(): assert cpu_power_path.exists(), f"CPU power data missing: {cpu_power_path}" # Test that we can actually read the CPU power data - cpu_power_data = ds.get_cpu_power_data() + cpu_power_data = ds.get_cpu_power_rows() assert isinstance(cpu_power_data, list), "CPU power data should be a list of rows" assert cpu_power_data, "CPU power data should not be empty"