Fix state pension (new vs basic SP), add fiscal_year to Simulation, add efrs to Python DATASETS

State pension now matches policyengine-uk approach: new SP recipients (reached SP age
after April 2016) get the full parameter rate directly; basic SP recipients get reported
amounts scaled by reform ratio. Also adds fiscal_year parameter to Simulation constructor
and adds "efrs" to the Python interface's DATASETS tuple for GCS download support.

Co-Authored-By: Claude <noreply@anthropic.com>

Author: nikhilwoodruff

interfaces/python/policyengine_uk_compiled/data.py

@@ -60,7 +60,7 @@ def _get_credentials() -> tuple[str, str]:
     return token.split(":", 1)
 
 
-DATASETS = ("frs", "lcfs", "spi", "was")
+DATASETS = ("frs", "efrs", "lcfs", "spi", "was")
 
 
 def ensure_dataset_year(dataset: str, year: int) -> Path:

src/data/lcfs.rs

@@ -181,20 +181,20 @@ pub fn load_lcfs(data_dir: &Path, fiscal_year: u32) -> anyhow::Result<Dataset> {
             weight,
             region,
             rent: rent_annual,
-            food_and_non_alcoholic_beverages: get_f64(hh_row, "p601").max(0.0) * WEEKS_IN_YEAR,
-            alcohol_and_tobacco:              get_f64(hh_row, "p602").max(0.0) * WEEKS_IN_YEAR,
-            clothing_and_footwear:            get_f64(hh_row, "p603").max(0.0) * WEEKS_IN_YEAR,
-            housing_water_and_fuel:           get_f64(hh_row, "p604").max(0.0) * WEEKS_IN_YEAR,
-            household_furnishings:            get_f64(hh_row, "p605").max(0.0) * WEEKS_IN_YEAR,
-            health:                           get_f64(hh_row, "p606").max(0.0) * WEEKS_IN_YEAR,
-            transport:                        get_f64(hh_row, "p607").max(0.0) * WEEKS_IN_YEAR,
-            communication:                    get_f64(hh_row, "p608").max(0.0) * WEEKS_IN_YEAR,
-            recreation_and_culture:           get_f64(hh_row, "p609").max(0.0) * WEEKS_IN_YEAR,
-            education:                        get_f64(hh_row, "p610").max(0.0) * WEEKS_IN_YEAR,
-            restaurants_and_hotels:           get_f64(hh_row, "p611").max(0.0) * WEEKS_IN_YEAR,
-            miscellaneous_goods_and_services: get_f64(hh_row, "p612").max(0.0) * WEEKS_IN_YEAR,
-            petrol_spending:                  get_f64(hh_row, "c72211").max(0.0) * WEEKS_IN_YEAR,
-            diesel_spending:                  get_f64(hh_row, "c72212").max(0.0) * WEEKS_IN_YEAR,
+            food_consumption:                          get_f64(hh_row, "p601").max(0.0) * WEEKS_IN_YEAR,
+            alcohol_tobacco_consumption:               get_f64(hh_row, "p602").max(0.0) * WEEKS_IN_YEAR,
+            clothing_consumption:                      get_f64(hh_row, "p603").max(0.0) * WEEKS_IN_YEAR,
+            housing_water_electricity_consumption:     get_f64(hh_row, "p604").max(0.0) * WEEKS_IN_YEAR,
+            furnishings_consumption:                   get_f64(hh_row, "p605").max(0.0) * WEEKS_IN_YEAR,
+            health_consumption:                        get_f64(hh_row, "p606").max(0.0) * WEEKS_IN_YEAR,
+            transport_consumption:                     get_f64(hh_row, "p607").max(0.0) * WEEKS_IN_YEAR,
+            communication_consumption:                 get_f64(hh_row, "p608").max(0.0) * WEEKS_IN_YEAR,
+            recreation_consumption:                    get_f64(hh_row, "p609").max(0.0) * WEEKS_IN_YEAR,
+            education_consumption:                     get_f64(hh_row, "p610").max(0.0) * WEEKS_IN_YEAR,
+            restaurants_consumption:                   get_f64(hh_row, "p611").max(0.0) * WEEKS_IN_YEAR,
+            miscellaneous_consumption:                 get_f64(hh_row, "p612").max(0.0) * WEEKS_IN_YEAR,
+            petrol_spending:                           get_f64(hh_row, "c72211").max(0.0) * WEEKS_IN_YEAR,
+            diesel_spending:                           get_f64(hh_row, "c72212").max(0.0) * WEEKS_IN_YEAR,
             ..Household::default()
         });
     }

src/data/was.rs

@@ -106,8 +106,8 @@ pub fn load_was(data_dir: &Path, fiscal_year: u32) -> anyhow::Result<Dataset> {
         // Wealth (zero if column absent in older waves)
         let financial_wealth = get_f64(row, &fin_wealth_col);
         let property_wealth  = get_f64(row, &prop_wealth_col);
-        let physical_wealth  = get_f64(row, &phys_wealth_col);
-        let total_wealth     = get_f64(row, &tot_wealth_col);
+        let _physical_wealth  = get_f64(row, &phys_wealth_col);
+        let _total_wealth     = get_f64(row, &tot_wealth_col);
 
         let hh_id = households.len();
         let bu_id = benunits.len();
@@ -177,10 +177,8 @@ pub fn load_was(data_dir: &Path, fiscal_year: u32) -> anyhow::Result<Dataset> {
             region,
             rent: rent_annual,
             council_tax,
-            financial_wealth,
+            net_financial_wealth: financial_wealth,
             property_wealth,
-            physical_wealth,
-            total_wealth,
             ..Household::default()
         });
     }

src/engine/simulation.rs

@@ -80,9 +80,10 @@ pub struct Simulation {
     pub benunits: Vec<BenUnit>,
     pub households: Vec<Household>,
     pub parameters: Parameters,
-    /// Baseline SP weekly rates for scaling reported amounts under reforms.
-    pub baseline_new_sp_weekly: f64,
+    /// Baseline old basic SP weekly rate for scaling reported amounts under reforms.
     pub baseline_old_sp_weekly: f64,
+    /// Fiscal year (e.g. 2025 for 2025/26) — used for new/basic SP cutoff.
+    pub fiscal_year: u32,
 }
 
 impl Simulation {
@@ -91,28 +92,28 @@ impl Simulation {
         benunits: Vec<BenUnit>,
         households: Vec<Household>,
         parameters: Parameters,
+        fiscal_year: u32,
     ) -> Self {
-        let baseline_new_sp_weekly = parameters.state_pension.new_state_pension_weekly;
         let baseline_old_sp_weekly = parameters.state_pension.old_basic_pension_weekly;
         Simulation {
             people, benunits, households, parameters,
-            baseline_new_sp_weekly, baseline_old_sp_weekly,
+            baseline_old_sp_weekly, fiscal_year,
         }
     }
 
-    /// Create a simulation with explicit baseline SP rates (for reform simulations
-    /// where the baseline rates differ from the reform parameters).
+    /// Create a simulation with explicit baseline old SP rate (for reform simulations
+    /// where the baseline rate differs from the reform parameters).
     pub fn new_with_baseline_sp(
         people: Vec<Person>,
         benunits: Vec<BenUnit>,
         households: Vec<Household>,
         parameters: Parameters,
-        baseline_new_sp_weekly: f64,
         baseline_old_sp_weekly: f64,
+        fiscal_year: u32,
     ) -> Self {
         Simulation {
             people, benunits, households, parameters,
-            baseline_new_sp_weekly, baseline_old_sp_weekly,
+            baseline_old_sp_weekly, fiscal_year,
         }
     }
 
@@ -138,13 +139,13 @@ impl Simulation {
         }
 
         // Phase 2: BenUnit-level calculations (parallelised)
-        let baseline_new_sp = self.baseline_new_sp_weekly;
         let baseline_old_sp = self.baseline_old_sp_weekly;
+        let fiscal_year = self.fiscal_year;
         let br: Vec<BenUnitResult> = self.benunits.par_iter().map(|bu| {
             let hh = &self.households[bu.household_id];
             variables::benefits::calculate_benunit(
                 bu, &self.people, &person_results, hh, &self.parameters,
-                baseline_new_sp, baseline_old_sp,
+                baseline_old_sp, fiscal_year,
             )
         }).collect();
         benunit_results = br;
@@ -299,16 +300,16 @@ mod tests {
 
         let bu = BenUnit {
             id: 0, household_id: 0, person_ids: vec![0, 1],
-            take_up_seed: 0.0, reported_cb: true,
+            migration_seed: 0.0, would_claim_cb: true,
             ..BenUnit::default()
         };
         let hh = Household {
             id: 0, person_ids: vec![0, 1], benunit_ids: vec![0],
             weight: 1.0, region: Region::London, council_tax: 1500.0,
-            ..Default::default()
+            ..Household::default()
         };
 
-        Simulation::new(vec![adult, child], vec![bu], vec![hh], params)
+        Simulation::new(vec![adult, child], vec![bu], vec![hh], params, 2025)
     }
 
     #[test]

src/main.rs

@@ -445,17 +445,18 @@ fn main() -> anyhow::Result<()> {
         dataset.benunits.clone(),
         dataset.households.clone(),
         baseline_params.clone(),
+        cli.year,
     );
     let baseline = baseline_sim.run();
 
-    // Run policy simulation (pass baseline SP rates so reported amounts scale correctly)
+    // Run policy simulation (pass baseline old SP rate so reported amounts scale correctly)
     let policy_sim = Simulation::new_with_baseline_sp(
         dataset.people.clone(),
         dataset.benunits.clone(),
         dataset.households.clone(),
         policy_params.clone(),
-        baseline_params.state_pension.new_state_pension_weekly,
         baseline_params.state_pension.old_basic_pension_weekly,
+        cli.year,
     );
     let reformed = policy_sim.run();
 
@@ -1021,7 +1022,7 @@ mod obr_validation {
         let params = Parameters::for_year(2025).unwrap();
         let sim = Simulation::new(
             dataset.people.clone(), dataset.benunits.clone(),
-            dataset.households.clone(), params,
+            dataset.households.clone(), params, 2025,
         );
         let results = sim.run();
 
@@ -1163,7 +1164,7 @@ mod historical_frs_tests {
 
             let sim = Simulation::new(
                 dataset.people.clone(), dataset.benunits.clone(),
-                dataset.households.clone(), params,
+                dataset.households.clone(), params, year,
             );
             let results = sim.run();