From ea1d4bc2df63047ccbdadbf6b80eddc45e91c061 Mon Sep 17 00:00:00 2001 From: Alex Dewar Date: Thu, 22 Jan 2026 15:37:19 +0000 Subject: [PATCH 1/8] Don't derive `Debug` for `FilePatch` We don't use it anywhere. --- src/patch.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/patch.rs b/src/patch.rs index 1f60c305..5bd711fe 100644 --- a/src/patch.rs +++ b/src/patch.rs @@ -109,7 +109,7 @@ impl ModelPatch { } /// Structure to hold patches for a model csv file. -#[derive(Debug, Clone)] +#[derive(Clone)] pub struct FilePatch { /// The file that this patch applies to (e.g. "agents.csv") filename: String, From 4809a10341460ed9742249620fa4d13c18f08b0e Mon Sep 17 00:00:00 2001 From: Alex Dewar Date: Thu, 22 Jan 2026 15:59:34 +0000 Subject: [PATCH 2/8] patch.rs: Add `CSVTable` alias --- src/patch.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/patch.rs b/src/patch.rs index 5bd711fe..7c28c0db 100644 --- a/src/patch.rs +++ b/src/patch.rs @@ -108,6 +108,11 @@ impl ModelPatch { } } +/// Represents all rows and columns of a CSV file. +/// +/// Assumes that each row is unique (as it should be for all MUSE2 input files). +type CSVTable = IndexSet>; + /// Structure to hold patches for a model csv file. #[derive(Clone)] pub struct FilePatch { @@ -116,9 +121,9 @@ pub struct FilePatch { /// The header row (optional). If `None`, the header is not checked against base files. header_row: Option>, /// Rows to delete (each row is a vector of fields) - to_delete: IndexSet>, + to_delete: CSVTable, /// Rows to add (each row is a vector of fields) - to_add: IndexSet>, + to_add: CSVTable, } impl FilePatch { @@ -229,7 +234,7 @@ fn modify_base_with_patch(base: &str, patch: &FilePatch) -> Result { } // Read all rows from the base, preserving order and checking for duplicates - let mut base_rows: IndexSet> = IndexSet::new(); + let mut base_rows: CSVTable = CSVTable::new(); for result in reader.records() { let record = result?; From 0142bae1fac7c056b936bab4df66f55dc37e6b08 Mon Sep 17 00:00:00 2001 From: Alex Dewar Date: Thu, 22 Jan 2026 16:03:57 +0000 Subject: [PATCH 3/8] FilePatch: Create `Replacements` enum with one variant --- src/patch.rs | 118 ++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 84 insertions(+), 34 deletions(-) diff --git a/src/patch.rs b/src/patch.rs index 7c28c0db..f6c60772 100644 --- a/src/patch.rs +++ b/src/patch.rs @@ -1,7 +1,7 @@ //! Code for applying patches to model input files. use anyhow::{Context, Result, ensure}; use csv::{ReaderBuilder, Trim, Writer}; -use indexmap::IndexSet; +use indexmap::{IndexSet, indexset}; use std::fs; use std::path::{Path, PathBuf}; @@ -113,6 +113,53 @@ impl ModelPatch { /// Assumes that each row is unique (as it should be for all MUSE2 input files). type CSVTable = IndexSet>; +/// The replacements to carry out for a file patch. +#[derive(Clone)] +enum Replacements { + /// Delete or add matching rows + Rows { + to_delete: CSVTable, + to_add: CSVTable, + }, +} + +impl Replacements { + fn apply(&self, rows: &mut CSVTable) -> Result<()> { + match self { + Self::Rows { to_delete, to_add } => { + // Check that there's no overlap between additions and deletions + for del_row in to_delete { + ensure!( + !to_add.contains(del_row), + "Row appears in both deletions and additions: {del_row:?}", + ); + } + + // Ensure every row requested for deletion actually exists in the base file. + for del_row in to_delete { + ensure!( + rows.contains(del_row), + "Row to delete not present in base file: {del_row:?}" + ); + } + + // Apply deletions + rows.retain(|row| !to_delete.contains(row)); + + // Apply additions (append to end, checking for duplicates) + for add_row in to_add { + ensure!( + rows.insert(add_row.clone()), + "Addition already present in base file: {add_row:?}" + ); + } + } + } + + Ok(()) + } +} + /// Structure to hold patches for a model csv file. #[derive(Clone)] pub struct FilePatch { @@ -120,10 +167,8 @@ pub struct FilePatch { filename: String, /// The header row (optional). If `None`, the header is not checked against base files. header_row: Option>, - /// Rows to delete (each row is a vector of fields) - to_delete: CSVTable, - /// Rows to add (each row is a vector of fields) - to_add: CSVTable, + /// The replacement(s) to carry out + replacements: Option, } impl FilePatch { @@ -132,8 +177,7 @@ impl FilePatch { FilePatch { filename: filename.into(), header_row: None, - to_delete: IndexSet::new(), - to_add: IndexSet::new(), + replacements: None, } } @@ -153,7 +197,20 @@ impl FilePatch { pub fn with_addition(mut self, row: impl Into) -> Self { let s = row.into(); let v = s.split(',').map(|s| s.trim().to_string()).collect(); - self.to_add.insert(v); + + match self.replacements { + Some(Replacements::Rows { + to_delete: _, + ref mut to_add, + }) => assert!(to_add.insert(v), "Attempted to add duplicate row: {s:?}"), + None => { + self.replacements = Some(Replacements::Rows { + to_delete: indexset![], + to_add: indexset![v], + }); + } + } + self } @@ -161,7 +218,23 @@ impl FilePatch { pub fn with_deletion(mut self, row: impl Into) -> Self { let s = row.into(); let v = s.split(',').map(|s| s.trim().to_string()).collect(); - self.to_delete.insert(v); + + match self.replacements { + Some(Replacements::Rows { + ref mut to_delete, + to_add: _, + }) => assert!( + to_delete.insert(v), + "Attempted to delete duplicate row: {s:?}" + ), + None => { + self.replacements = Some(Replacements::Rows { + to_delete: indexset![v], + to_add: indexset![], + }); + } + } + self } @@ -251,31 +324,8 @@ fn modify_base_with_patch(base: &str, patch: &FilePatch) -> Result { ); } - // Check that there's no overlap between additions and deletions - for del_row in &patch.to_delete { - ensure!( - !patch.to_add.contains(del_row), - "Row appears in both deletions and additions: {del_row:?}", - ); - } - - // Ensure every row requested for deletion actually exists in the base file. - for del_row in &patch.to_delete { - ensure!( - base_rows.contains(del_row), - "Row to delete not present in base file: {del_row:?}" - ); - } - - // Apply deletions - base_rows.retain(|row| !patch.to_delete.contains(row)); - - // Apply additions (append to end, checking for duplicates) - for add_row in &patch.to_add { - ensure!( - base_rows.insert(add_row.clone()), - "Addition already present in base file: {add_row:?}" - ); + if let Some(replacements) = &patch.replacements { + replacements.apply(&mut base_rows)?; } // Serialize CSV output using csv::Writer From 4e9b52d7c99772c8f26802ede10c9ef30e0f2708 Mon Sep 17 00:00:00 2001 From: Alex Dewar Date: Thu, 22 Jan 2026 16:12:03 +0000 Subject: [PATCH 4/8] Make `map-macro` into a non-dev dependency --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 45bc85b1..c9c8c62c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,10 +38,10 @@ documented = "0.9.2" dirs = "6.0.0" edit = "0.1.5" erased-serde = "0.4.9" +map-macro = "0.3.0" [dev-dependencies] assert_cmd = "2.1.2" -map-macro = "0.3.0" rstest = {version = "0.26.1", default-features = false, features = ["crate-name"]} yaml-rust2 = {version = "0.11.0", default-features = false} From 0cfa9b647388cccf6f75f0e9687a13e634a3c267 Mon Sep 17 00:00:00 2001 From: Alex Dewar Date: Thu, 22 Jan 2026 16:28:09 +0000 Subject: [PATCH 5/8] FilePatch: Add option to replace single values Closes #1090. --- src/patch.rs | 136 ++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 101 insertions(+), 35 deletions(-) diff --git a/src/patch.rs b/src/patch.rs index f6c60772..98819516 100644 --- a/src/patch.rs +++ b/src/patch.rs @@ -2,6 +2,8 @@ use anyhow::{Context, Result, ensure}; use csv::{ReaderBuilder, Trim, Writer}; use indexmap::{IndexSet, indexset}; +use map_macro::hash_map; +use std::collections::HashMap; use std::fs; use std::path::{Path, PathBuf}; @@ -114,45 +116,66 @@ impl ModelPatch { type CSVTable = IndexSet>; /// The replacements to carry out for a file patch. -#[derive(Clone)] +#[derive(Clone, strum::Display)] enum Replacements { /// Delete or add matching rows Rows { to_delete: CSVTable, to_add: CSVTable, }, + /// Find and replace single values + Values { to_replace: HashMap }, } -impl Replacements { - fn apply(&self, rows: &mut CSVTable) -> Result<()> { - match self { - Self::Rows { to_delete, to_add } => { - // Check that there's no overlap between additions and deletions - for del_row in to_delete { - ensure!( - !to_add.contains(del_row), - "Row appears in both deletions and additions: {del_row:?}", - ); - } +fn replace_rows(rows: &mut CSVTable, to_add: &CSVTable, to_delete: &CSVTable) -> Result<()> { + // Check that there's no overlap between additions and deletions + for del_row in to_delete { + ensure!( + !to_add.contains(del_row), + "Row appears in both deletions and additions: {del_row:?}", + ); + } - // Ensure every row requested for deletion actually exists in the base file. - for del_row in to_delete { - ensure!( - rows.contains(del_row), - "Row to delete not present in base file: {del_row:?}" - ); - } + // Ensure every row requested for deletion actually exists in the base file. + for del_row in to_delete { + ensure!( + rows.contains(del_row), + "Row to delete not present in base file: {del_row:?}" + ); + } - // Apply deletions - rows.retain(|row| !to_delete.contains(row)); + // Apply deletions + rows.retain(|row| !to_delete.contains(row)); - // Apply additions (append to end, checking for duplicates) - for add_row in to_add { - ensure!( - rows.insert(add_row.clone()), - "Addition already present in base file: {add_row:?}" - ); + // Apply additions (append to end, checking for duplicates) + for add_row in to_add { + ensure!( + rows.insert(add_row.clone()), + "Addition already present in base file: {add_row:?}" + ); + } + + Ok(()) +} + +impl Replacements { + fn apply(&self, rows: &mut CSVTable) -> Result<()> { + match self { + Self::Rows { to_delete, to_add } => replace_rows(rows, to_add, to_delete)?, + Self::Values { to_replace } => { + let mut to_add = CSVTable::new(); + let mut to_delete = CSVTable::new(); + for row in rows.iter() { + if row.iter().any(|value| to_replace.contains_key(value)) { + to_delete.insert(row.clone()); + let new_row = row + .iter() + .map(|old_value| to_replace.get(old_value).unwrap_or(old_value).clone()) + .collect(); + to_add.insert(new_row); + } } + replace_rows(rows, &to_add, &to_delete)?; } } @@ -199,16 +222,17 @@ impl FilePatch { let v = s.split(',').map(|s| s.trim().to_string()).collect(); match self.replacements { - Some(Replacements::Rows { - to_delete: _, - ref mut to_add, - }) => assert!(to_add.insert(v), "Attempted to add duplicate row: {s:?}"), None => { self.replacements = Some(Replacements::Rows { to_delete: indexset![], to_add: indexset![v], }); } + Some(Replacements::Rows { + to_delete: _, + ref mut to_add, + }) => assert!(to_add.insert(v), "Attempted to add duplicate row: {s:?}"), + Some(r) => panic!("Cannot add rows when replacement type is already set to: {r}"), } self @@ -220,6 +244,12 @@ impl FilePatch { let v = s.split(',').map(|s| s.trim().to_string()).collect(); match self.replacements { + None => { + self.replacements = Some(Replacements::Rows { + to_delete: indexset![v], + to_add: indexset![], + }); + } Some(Replacements::Rows { ref mut to_delete, to_add: _, @@ -227,12 +257,31 @@ impl FilePatch { to_delete.insert(v), "Attempted to delete duplicate row: {s:?}" ), + Some(r) => panic!("Cannot delete rows when replacement type is already set to: {r}"), + } + + self + } + + /// Replace a value if found in any field + pub fn with_replace_value( + mut self, + old_value: impl Into, + new_value: impl Into, + ) -> Self { + let old_value = old_value.into(); + let new_value = new_value.into(); + match self.replacements { None => { - self.replacements = Some(Replacements::Rows { - to_delete: indexset![v], - to_add: indexset![], + self.replacements = Some(Replacements::Values { + to_replace: hash_map! { old_value => new_value }, }); } + Some(Replacements::Values { ref mut to_replace }) => assert!( + to_replace.insert(old_value, new_value).is_none(), + "Attempted to replace same value multiple times" + ), + Some(r) => panic!("Cannot replace values when replacement type is already set to: {r}"), } self @@ -410,7 +459,7 @@ mod tests { } #[test] - fn file_patch() { + fn file_patch_row() { // Patch with a small change to an asset capacity let assets_patch = FilePatch::new("assets.csv") .with_deletion("GASDRV,GBR,A0_GEX,4002.26,2020") @@ -429,6 +478,23 @@ mod tests { assert!(assets_content.contains("GASDRV,GBR,A0_GEX,4003.26,2020")); } + #[test] + fn file_patch_value() { + // Replace decision rule with a different (made-up) one + let agents_patch = FilePatch::new("agents.csv").with_replace_value("single", "madeup"); + + // Build patched model into a temporary directory + let model_dir = ModelPatch::from_example("simple") + .with_file_patch(agents_patch) + .build_to_tempdir() + .unwrap(); + + let agents_path = model_dir.path().join("agents.csv"); + let agents_content = std::fs::read_to_string(agents_path).unwrap(); + assert!(!agents_content.contains("simple")); + assert!(agents_content.contains("madeup")); + } + #[test] fn toml_patch() { // Patch to add an extra milestone year (2050) From 4f18bcecd81f6b4fd6d1effe52f46b7ebd7d08ff Mon Sep 17 00:00:00 2001 From: Alex Dewar Date: Thu, 15 Jan 2026 16:54:07 +0000 Subject: [PATCH 6/8] Unhide NPV option --- src/input/agent/objective.rs | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/src/input/agent/objective.rs b/src/input/agent/objective.rs index a8e5c8be..7c036910 100644 --- a/src/input/agent/objective.rs +++ b/src/input/agent/objective.rs @@ -1,13 +1,10 @@ //! Code for reading agent objectives from a CSV file. use super::super::{input_err_msg, read_csv, try_insert}; -use crate::ISSUES_URL; use crate::agent::{AgentID, AgentMap, AgentObjectiveMap, DecisionRule, ObjectiveType}; -use crate::model::{ALLOW_BROKEN_OPTION_NAME, broken_model_options_allowed}; use crate::units::Dimensionless; use crate::year::parse_year_str; use anyhow::{Context, Result, ensure}; use itertools::Itertools; -use log::warn; use serde::Deserialize; use std::collections::HashMap; use std::path::Path; @@ -93,24 +90,6 @@ where missing_years.is_empty(), "Agent {agent_id} is missing objectives for the following milestone years: {missing_years:?}" ); - - let npv_years = milestone_years - .iter() - .copied() - .filter(|year| agent_objectives[year] == ObjectiveType::NetPresentValue) - .collect_vec(); - if !npv_years.is_empty() { - ensure!( - broken_model_options_allowed(), - "The NPV option is BROKEN and should not be used. See: {ISSUES_URL}/716.\n\ - If you are sure that you want to enable it anyway, you need to set the \ - {ALLOW_BROKEN_OPTION_NAME} option to true." - ); - warn!( - "Agent {agent_id} is using NPV in years {npv_years:?}. \ - The NPV option is BROKEN and should not be used. See: {ISSUES_URL}/716." - ); - } } Ok(all_objectives) From ab6162f5a8fa550cd89d431eadd244907fd30579 Mon Sep 17 00:00:00 2001 From: Alex Dewar Date: Thu, 22 Jan 2026 16:37:29 +0000 Subject: [PATCH 7/8] Add patched example using NPV and add regression test Closes #1062. --- src/example/patches.rs | 25 ++++++++++++++++--------- tests/regression.rs | 1 + 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/example/patches.rs b/src/example/patches.rs index 580a6a6c..576118d6 100644 --- a/src/example/patches.rs +++ b/src/example/patches.rs @@ -13,15 +13,22 @@ static PATCHES: LazyLock = LazyLock::new(get_all_patches); /// Get all patches fn get_all_patches() -> PatchMap { - [( - // The simple example with gas boiler process made divisible - "simple_divisible", - vec![ - FilePatch::new("processes.csv") - .with_deletion("RGASBR,Gas boiler,all,RSHEAT,2020,2040,1.0,") - .with_addition("RGASBR,Gas boiler,all,RSHEAT,2020,2040,1.0,1000"), - ], - )] + [ + ( + // The simple example with gas boiler process made divisible + "simple_divisible", + vec![ + FilePatch::new("processes.csv") + .with_deletion("RGASBR,Gas boiler,all,RSHEAT,2020,2040,1.0,") + .with_addition("RGASBR,Gas boiler,all,RSHEAT,2020,2040,1.0,1000"), + ], + ), + // The simple example with the objective type set to NPV + ( + "simple_npv", + vec![FilePatch::new("agent_objectives.csv").with_replace_value("lcox", "npv")], + ), + ] .into_iter() .collect() } diff --git a/tests/regression.rs b/tests/regression.rs index e4dd53b6..b60ac228 100644 --- a/tests/regression.rs +++ b/tests/regression.rs @@ -25,6 +25,7 @@ define_regression_test!(circularity); // Patched examples define_regression_test_with_patches!(simple_divisible); +define_regression_test_with_patches!(simple_npv); // ------ END: regression tests ------ From d4c279f1ac900fff32a336badb6d92f1e5c02a3c Mon Sep 17 00:00:00 2001 From: Alex Dewar Date: Fri, 23 Jan 2026 07:53:01 +0000 Subject: [PATCH 8/8] patch.rs: Fix mistake in test Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/patch.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/patch.rs b/src/patch.rs index 98819516..571b4770 100644 --- a/src/patch.rs +++ b/src/patch.rs @@ -491,7 +491,7 @@ mod tests { let agents_path = model_dir.path().join("agents.csv"); let agents_content = std::fs::read_to_string(agents_path).unwrap(); - assert!(!agents_content.contains("simple")); + assert!(!agents_content.contains("single")); assert!(agents_content.contains("madeup")); }