Drop formulaic SPM outputs from exported H5s

changelog.d/954.fixed.md

@@ -0,0 +1 @@
+Drop formulaic SPM poverty outputs from cloned and local-area H5 exports.

policyengine_us_data/calibration/entity_clone.py

@@ -20,6 +20,9 @@
 from policyengine_us_data.calibration.block_assignment import (
     derive_geography_from_blocks,
 )
+from policyengine_us_data.calibration.formulaic_inputs import (
+    drop_formulaic_spm_inputs,
+)
 from policyengine_us_data.datasets.puf.variable_roles import (
     PUF_REPORTED_CALCULATED_TAX_OUTPUT_VARIABLES,
 )
@@ -270,7 +273,7 @@ def materialize_clone_household_chunk(
     vars_to_save = (
         set(sim.input_variables) - PUF_REPORTED_CALCULATED_TAX_OUTPUT_VARIABLES
     )
-    vars_to_save.discard("spm_unit_spm_threshold")
+    drop_formulaic_spm_inputs(vars_to_save)
     vars_to_save.add("county")
     vars_to_save.add("congressional_district_geoid")
     for geo_var in [

policyengine_us_data/calibration/formulaic_inputs.py

@@ -0,0 +1,19 @@
+"""Formula outputs that must not be persisted as dataset leaf inputs."""
+
+FORMULAIC_SPM_INPUTS_TO_DROP = frozenset(
+    {
+        "person_in_poverty",
+        "in_poverty",
+        "in_deep_poverty",
+        "spm_unit_is_in_spm_poverty",
+        "spm_unit_is_in_deep_spm_poverty",
+        "spm_unit_spm_threshold",
+        "spm_unit_geographic_adjustment",
+    }
+)
+
+
+def drop_formulaic_spm_inputs(variable_names: set[str]) -> None:
+    """Remove SPM formula outputs from a mutable variable-name set."""
+
+    variable_names.difference_update(FORMULAIC_SPM_INPUTS_TO_DROP)

policyengine_us_data/calibration/publish_local_area.py

@@ -34,6 +34,9 @@
 from policyengine_us_data.calibration.block_assignment import (
     derive_geography_from_blocks,
 )
+from policyengine_us_data.calibration.formulaic_inputs import (
+    drop_formulaic_spm_inputs,
+)
 from policyengine_us_data.datasets.puf.variable_roles import (
     PUF_REPORTED_CALCULATED_TAX_OUTPUT_VARIABLES,
 )
@@ -517,7 +520,7 @@ def build_h5(
     vars_to_save = (
         set(sim.input_variables) - PUF_REPORTED_CALCULATED_TAX_OUTPUT_VARIABLES
     )
-    vars_to_save.discard("spm_unit_spm_threshold")
+    drop_formulaic_spm_inputs(vars_to_save)
     vars_to_save.add("county")
     vars_to_save.add("congressional_district_geoid")
     for gv in [

tests/unit/calibration/test_entity_clone.py

@@ -110,7 +110,15 @@ def test_materialize_clone_household_chunk_drops_legacy_spm_threshold_input(
     fixture_entity_maps,
 ):
     original_input_variables = list(fixture_sim.input_variables)
-    fixture_sim.input_variables.append("spm_unit_spm_threshold")
+    fixture_sim.input_variables.extend(
+        [
+            "person_in_poverty",
+            "in_poverty",
+            "in_deep_poverty",
+            "spm_unit_spm_threshold",
+            "spm_unit_geographic_adjustment",
+        ]
+    )
     monkeypatch.setattr(
         "policyengine_us_data.calibration.entity_clone.derive_geography_from_blocks",
         _fake_geography_from_blocks,
@@ -131,6 +139,9 @@ def test_materialize_clone_household_chunk_drops_legacy_spm_threshold_input(
         fixture_sim.input_variables = original_input_variables
 
     with h5py.File(output_path, "r") as h5:
+        assert "person_in_poverty" not in h5
+        assert "in_poverty" not in h5
+        assert "in_deep_poverty" not in h5
         assert "spm_unit_spm_threshold" not in h5
         assert "spm_unit_geographic_adjustment" not in h5
 

tests/unit/calibration/test_formulaic_inputs.py

@@ -0,0 +1,23 @@
+from policyengine_us_data.calibration.formulaic_inputs import (
+    FORMULAIC_SPM_INPUTS_TO_DROP,
+    drop_formulaic_spm_inputs,
+)
+
+
+def test_drop_formulaic_spm_inputs_removes_poverty_formula_outputs():
+    variable_names = {
+        "person_in_poverty",
+        "in_poverty",
+        "in_deep_poverty",
+        "spm_unit_spm_threshold",
+        "spm_unit_geographic_adjustment",
+        "household_weight",
+    }
+
+    drop_formulaic_spm_inputs(variable_names)
+
+    assert variable_names == {"household_weight"}
+
+
+def test_formulaic_spm_inputs_includes_person_poverty():
+    assert "person_in_poverty" in FORMULAIC_SPM_INPUTS_TO_DROP

tests/unit/test_extended_cps.py

@@ -158,6 +158,7 @@ def test_spm_threshold_is_formula_output_not_qrf_imputed(self):
         assert "spm_unit_spm_threshold" not in set(CPS_ONLY_IMPUTED_VARIABLES)
         assert "spm_unit_spm_threshold" not in ExtendedCPS._keep_formula_vars()
         assert "spm_unit_geographic_adjustment" not in ExtendedCPS._keep_formula_vars()
+        assert "person_in_poverty" not in ExtendedCPS._keep_formula_vars()
 
     def test_weeks_worked_is_preserved_for_future_year_formulas(self):
         assert "weeks_worked" in ExtendedCPS._keep_formula_vars()