Warn when a dataset carries columns that match no variable

changelog.d/dataset-unknown-column-warning.added.md

@@ -0,0 +1 @@
+Warn when a dataset contains columns that do not match any variable in the tax-benefit system, instead of silently ignoring them.

policyengine_core/simulations/simulation.py

@@ -467,6 +467,7 @@ def get_eternity_array(name):
             # Ensure we're back to all person-level data.
             data = data_copy
 
+        unknown_columns = []
         if self.dataset.data_format != Dataset.FLAT_FILE:
             for variable in data:
                 if variable in self.tax_benefit_system.variables:
@@ -482,8 +483,7 @@ def get_eternity_array(name):
                             variable, self.dataset.time_period, data[variable]
                         )
                 else:
-                    # Silently skip.
-                    pass
+                    unknown_columns.append(variable)
         else:
             for variable in data:
                 if "__" in variable:
@@ -493,6 +493,7 @@ def get_eternity_array(name):
                     time_period = self.dataset.time_period or self.default_input_period
 
                 if variable_name not in self.tax_benefit_system.variables:
+                    unknown_columns.append(variable)
                     continue
 
                 variable_meta = self.tax_benefit_system.get_variable(variable_name)
@@ -510,6 +511,20 @@ def get_eternity_array(name):
 
                 self.set_input(variable_name, time_period, entity_level_data)
 
+        if unknown_columns:
+            # A skipped column usually means the dataset was built for a
+            # different model version (e.g. an input variable was renamed or
+            # removed), and its data is silently lost — say so instead of
+            # loading as if nothing happened.
+            shown = ", ".join(sorted(unknown_columns)[:10])
+            if len(unknown_columns) > 10:
+                shown += f", … ({len(unknown_columns) - 10} more)"
+            logging.warning(
+                f"The dataset contains {len(unknown_columns)} column(s) that "
+                f"do not match any variable in the tax-benefit system and "
+                f"were ignored: {shown}"
+            )
+
         self.default_calculation_period = (
             self.dataset.time_period or self.default_calculation_period
         )

tests/core/test_dataset_unknown_column_warning.py

@@ -0,0 +1,61 @@
+import logging
+
+import numpy as np
+import pandas as pd
+
+from policyengine_core.country_template import Microsimulation
+from policyengine_core.data import Dataset
+
+
+def test__given_unknown_dataset_column__then_warns_and_still_loads(caplog):
+    # Given a dataset carrying a column that matches no variable (e.g. an
+    # input that was renamed or removed from the model after the dataset
+    # was built).
+    data = {
+        "person_id__2022": [0, 1, 2],
+        "person_household_id__2022": [0, 0, 1],
+        "person_household_role__2022": ["parent", "child", "parent"],
+        "household_weight__2022": [10.0, 10.0, 20.0],
+        "salary__2022-01": [100.0, 200.0, 300.0],
+        "a_removed_input__2022": [1.0, 2.0, 3.0],
+    }
+
+    # When the simulation is built from it
+    with caplog.at_level(logging.WARNING):
+        simulation = Microsimulation(
+            dataset=Dataset.from_dataframe(pd.DataFrame(data), "2022")
+        )
+        salary = simulation.calculate("salary", "2022-01")
+
+    # Then the unknown column is reported instead of vanishing silently
+    warnings = [
+        record.message
+        for record in caplog.records
+        if "a_removed_input__2022" in record.message
+    ]
+    assert warnings, "expected a warning naming the ignored column"
+    assert "ignored" in warnings[0]
+
+    # And the known columns still load normally
+    np.testing.assert_array_equal(salary.values, np.array([100.0, 200.0, 300.0]))
+
+
+def test__given_only_known_columns__then_no_unknown_column_warning(caplog):
+    data = {
+        "person_id__2022": [0, 1],
+        "person_household_id__2022": [0, 0],
+        "person_household_role__2022": ["parent", "child"],
+        "household_weight__2022": [10.0, 10.0],
+        "salary__2022-01": [100.0, 200.0],
+    }
+
+    with caplog.at_level(logging.WARNING):
+        Microsimulation(
+            dataset=Dataset.from_dataframe(pd.DataFrame(data), "2022")
+        ).calculate("salary", "2022-01")
+
+    assert not [
+        record
+        for record in caplog.records
+        if "do not match any variable" in record.message
+    ]