From e2f04e1239aaa8815bbca35e3f547d96f8677141 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 14 Aug 2026 10:57:51 -0400 Subject: [PATCH] Warn when a dataset carries columns that match no variable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #530. Both dataset-load branches in Simulation.build_from_dataset silently dropped columns whose variable name is not in the tax-benefit system — one carried an explicit "Silently skip" comment. That makes any country-model input removal or rename silently zero the affected data for every existing dataset (the blocker for policyengine-us#9275's deprecated-input removals). Collect the skipped column names in both branches and emit one aggregated warning naming them (first ten, plus a count), so a version-mismatched dataset announces itself instead of loading as if nothing happened. Known columns load exactly as before, and clean datasets stay quiet — both covered by new tests on the flat-file path. Co-Authored-By: Claude Fable 5 --- .../dataset-unknown-column-warning.added.md | 1 + policyengine_core/simulations/simulation.py | 19 +++++- .../test_dataset_unknown_column_warning.py | 61 +++++++++++++++++++ 3 files changed, 79 insertions(+), 2 deletions(-) create mode 100644 changelog.d/dataset-unknown-column-warning.added.md create mode 100644 tests/core/test_dataset_unknown_column_warning.py diff --git a/changelog.d/dataset-unknown-column-warning.added.md b/changelog.d/dataset-unknown-column-warning.added.md new file mode 100644 index 00000000..a48022d3 --- /dev/null +++ b/changelog.d/dataset-unknown-column-warning.added.md @@ -0,0 +1 @@ +Warn when a dataset contains columns that do not match any variable in the tax-benefit system, instead of silently ignoring them. diff --git a/policyengine_core/simulations/simulation.py b/policyengine_core/simulations/simulation.py index 30fdfcff..0d4e001c 100644 --- a/policyengine_core/simulations/simulation.py +++ b/policyengine_core/simulations/simulation.py @@ -467,6 +467,7 @@ def get_eternity_array(name): # Ensure we're back to all person-level data. data = data_copy + unknown_columns = [] if self.dataset.data_format != Dataset.FLAT_FILE: for variable in data: if variable in self.tax_benefit_system.variables: @@ -482,8 +483,7 @@ def get_eternity_array(name): variable, self.dataset.time_period, data[variable] ) else: - # Silently skip. - pass + unknown_columns.append(variable) else: for variable in data: if "__" in variable: @@ -493,6 +493,7 @@ def get_eternity_array(name): time_period = self.dataset.time_period or self.default_input_period if variable_name not in self.tax_benefit_system.variables: + unknown_columns.append(variable) continue variable_meta = self.tax_benefit_system.get_variable(variable_name) @@ -510,6 +511,20 @@ def get_eternity_array(name): self.set_input(variable_name, time_period, entity_level_data) + if unknown_columns: + # A skipped column usually means the dataset was built for a + # different model version (e.g. an input variable was renamed or + # removed), and its data is silently lost — say so instead of + # loading as if nothing happened. + shown = ", ".join(sorted(unknown_columns)[:10]) + if len(unknown_columns) > 10: + shown += f", … ({len(unknown_columns) - 10} more)" + logging.warning( + f"The dataset contains {len(unknown_columns)} column(s) that " + f"do not match any variable in the tax-benefit system and " + f"were ignored: {shown}" + ) + self.default_calculation_period = ( self.dataset.time_period or self.default_calculation_period ) diff --git a/tests/core/test_dataset_unknown_column_warning.py b/tests/core/test_dataset_unknown_column_warning.py new file mode 100644 index 00000000..7450b92d --- /dev/null +++ b/tests/core/test_dataset_unknown_column_warning.py @@ -0,0 +1,61 @@ +import logging + +import numpy as np +import pandas as pd + +from policyengine_core.country_template import Microsimulation +from policyengine_core.data import Dataset + + +def test__given_unknown_dataset_column__then_warns_and_still_loads(caplog): + # Given a dataset carrying a column that matches no variable (e.g. an + # input that was renamed or removed from the model after the dataset + # was built). + data = { + "person_id__2022": [0, 1, 2], + "person_household_id__2022": [0, 0, 1], + "person_household_role__2022": ["parent", "child", "parent"], + "household_weight__2022": [10.0, 10.0, 20.0], + "salary__2022-01": [100.0, 200.0, 300.0], + "a_removed_input__2022": [1.0, 2.0, 3.0], + } + + # When the simulation is built from it + with caplog.at_level(logging.WARNING): + simulation = Microsimulation( + dataset=Dataset.from_dataframe(pd.DataFrame(data), "2022") + ) + salary = simulation.calculate("salary", "2022-01") + + # Then the unknown column is reported instead of vanishing silently + warnings = [ + record.message + for record in caplog.records + if "a_removed_input__2022" in record.message + ] + assert warnings, "expected a warning naming the ignored column" + assert "ignored" in warnings[0] + + # And the known columns still load normally + np.testing.assert_array_equal(salary.values, np.array([100.0, 200.0, 300.0])) + + +def test__given_only_known_columns__then_no_unknown_column_warning(caplog): + data = { + "person_id__2022": [0, 1], + "person_household_id__2022": [0, 0], + "person_household_role__2022": ["parent", "child"], + "household_weight__2022": [10.0, 10.0], + "salary__2022-01": [100.0, 200.0], + } + + with caplog.at_level(logging.WARNING): + Microsimulation( + dataset=Dataset.from_dataframe(pd.DataFrame(data), "2022") + ).calculate("salary", "2022-01") + + assert not [ + record + for record in caplog.records + if "do not match any variable" in record.message + ]