From 900f7dd7b4e6bba96ad7cde0d6e8550795333796 Mon Sep 17 00:00:00 2001 From: David Berenstein Date: Wed, 12 Aug 2026 18:08:01 +0200 Subject: [PATCH] fix: drop dtype coercion in CSV update path Updating a run row coerced each incoming value to the dtype pandas inferred for the existing column. Columns empty in every row are read back as float64, so numpy.float64("") / numpy.float64(None) raised. Rebuild the row via concat instead, which dedupes by run_id without touching dtypes. Co-Authored-By: Claude Opus 5 (1M context) --- codecarbon/output_methods/file.py | 15 ++++----- tests/output_methods/test_file.py | 56 +++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 9 deletions(-) diff --git a/codecarbon/output_methods/file.py b/codecarbon/output_methods/file.py index 6a13d5b41..76eebf417 100644 --- a/codecarbon/output_methods/file.py +++ b/codecarbon/output_methods/file.py @@ -107,9 +107,7 @@ def out(self, total: EmissionsData, _): else: df = pd.read_csv(self.save_file_path) df_run = df.loc[df.run_id == total.run_id] - if len(df_run) < 1: - df = pd.concat([df, new_df]) - elif len(df_run) > 1: + if len(df_run) > 1: logger.warning( f"CSV contains more than 1 ({len(df_run)})" + f" rows with current run ID ({total.run_id})." @@ -117,12 +115,11 @@ def out(self, total: EmissionsData, _): ) df = pd.concat([df, new_df]) else: - update_values = {} - for col, val in dict(total.values).items(): - update_values[col] = df[col].dtype.type(val) - df.loc[df.run_id == total.run_id, update_values.keys()] = ( - update_values.values() - ) + # Drop the previous row for this run (if any) and re-append it. + # Assigning column by column would coerce values to the dtype + # pandas inferred for the existing column, which breaks for + # columns that are empty in every row (read back as float64). + df = pd.concat([df.loc[df.run_id != total.run_id], new_df]) df.to_csv(self.save_file_path, index=False) def task_out(self, data: List[TaskEmissionsData], experiment_name: str): diff --git a/tests/output_methods/test_file.py b/tests/output_methods/test_file.py index e8bccfdf0..0b7f301e4 100644 --- a/tests/output_methods/test_file.py +++ b/tests/output_methods/test_file.py @@ -169,6 +169,62 @@ def test_file_output_out_update_file_exists_one_matchingrows(self): df = pd.read_csv(os.path.join(self.temp_dir, "test.csv")) self.assertEqual(df["cpu_power"].iloc[0], 2) + def test_file_output_out_update_with_always_empty_columns(self): + """Regression test: updating a run must not coerce incoming values to the + dtype pandas inferred for the existing column. + + An OfflineEmissionsTracker leaves longitude/latitude empty, and + gpu_count/gpu_model are empty on CPU-only machines. Such columns are read + back from the CSV as float64, so the previous implementation evaluated + numpy.float64("") / numpy.float64(None) and raised on the second write. + """ + empty_columns_data = EmissionsData( + timestamp="2023-01-01T00:00:00", + project_name="test_project", + run_id="test_run_id", + experiment_id="test_experiment_id", + duration=10, + emissions=0.5, + emissions_rate=0.05, + cpu_power=20, + gpu_power=0, + ram_power=5, + cpu_energy=200, + gpu_energy=0, + ram_energy=50, + energy_consumed=250, + water_consumed=0.1, + country_name="Testland", + country_iso_code="TS", + region="Test Region", + cloud_provider="", + cloud_region="", + os="TestOS", + python_version="3.8", + codecarbon_version="2.0", + cpu_count=4, + cpu_model="Test CPU", + gpu_count=None, + gpu_model=None, + longitude="", + latitude="", + ram_total_size=16, + tracking_mode="machine", + ) + + file_output = FileOutput("test.csv", self.temp_dir, on_csv_write="update") + file_output.out(empty_columns_data, None) + + empty_columns_data.cpu_power = 2 + # This should not raise. + file_output.out(empty_columns_data, None) + + df = pd.read_csv(os.path.join(self.temp_dir, "test.csv")) + self.assertEqual(len(df), 1) + self.assertEqual(df["cpu_power"].iloc[0], 2) + self.assertIn("longitude", df.columns) + self.assertIn("gpu_model", df.columns) + # def test_file_output_out_consistent_column_ordering(self): # file_output = FileOutput("test.csv", self.temp_dir, on_csv_write="append") # file_output.out(self.emissions_data, None)