diff --git a/Cargo.toml b/Cargo.toml index 45bc85b1..c9c8c62c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,10 +38,10 @@ documented = "0.9.2" dirs = "6.0.0" edit = "0.1.5" erased-serde = "0.4.9" +map-macro = "0.3.0" [dev-dependencies] assert_cmd = "2.1.2" -map-macro = "0.3.0" rstest = {version = "0.26.1", default-features = false, features = ["crate-name"]} yaml-rust2 = {version = "0.11.0", default-features = false} diff --git a/src/example/patches.rs b/src/example/patches.rs index 580a6a6c..576118d6 100644 --- a/src/example/patches.rs +++ b/src/example/patches.rs @@ -13,15 +13,22 @@ static PATCHES: LazyLock = LazyLock::new(get_all_patches); /// Get all patches fn get_all_patches() -> PatchMap { - [( - // The simple example with gas boiler process made divisible - "simple_divisible", - vec![ - FilePatch::new("processes.csv") - .with_deletion("RGASBR,Gas boiler,all,RSHEAT,2020,2040,1.0,") - .with_addition("RGASBR,Gas boiler,all,RSHEAT,2020,2040,1.0,1000"), - ], - )] + [ + ( + // The simple example with gas boiler process made divisible + "simple_divisible", + vec![ + FilePatch::new("processes.csv") + .with_deletion("RGASBR,Gas boiler,all,RSHEAT,2020,2040,1.0,") + .with_addition("RGASBR,Gas boiler,all,RSHEAT,2020,2040,1.0,1000"), + ], + ), + // The simple example with the objective type set to NPV + ( + "simple_npv", + vec![FilePatch::new("agent_objectives.csv").with_replace_value("lcox", "npv")], + ), + ] .into_iter() .collect() } diff --git a/src/input/agent/objective.rs b/src/input/agent/objective.rs index a8e5c8be..7c036910 100644 --- a/src/input/agent/objective.rs +++ b/src/input/agent/objective.rs @@ -1,13 +1,10 @@ //! Code for reading agent objectives from a CSV file. use super::super::{input_err_msg, read_csv, try_insert}; -use crate::ISSUES_URL; use crate::agent::{AgentID, AgentMap, AgentObjectiveMap, DecisionRule, ObjectiveType}; -use crate::model::{ALLOW_BROKEN_OPTION_NAME, broken_model_options_allowed}; use crate::units::Dimensionless; use crate::year::parse_year_str; use anyhow::{Context, Result, ensure}; use itertools::Itertools; -use log::warn; use serde::Deserialize; use std::collections::HashMap; use std::path::Path; @@ -93,24 +90,6 @@ where missing_years.is_empty(), "Agent {agent_id} is missing objectives for the following milestone years: {missing_years:?}" ); - - let npv_years = milestone_years - .iter() - .copied() - .filter(|year| agent_objectives[year] == ObjectiveType::NetPresentValue) - .collect_vec(); - if !npv_years.is_empty() { - ensure!( - broken_model_options_allowed(), - "The NPV option is BROKEN and should not be used. See: {ISSUES_URL}/716.\n\ - If you are sure that you want to enable it anyway, you need to set the \ - {ALLOW_BROKEN_OPTION_NAME} option to true." - ); - warn!( - "Agent {agent_id} is using NPV in years {npv_years:?}. \ - The NPV option is BROKEN and should not be used. See: {ISSUES_URL}/716." - ); - } } Ok(all_objectives) diff --git a/src/patch.rs b/src/patch.rs index 1f60c305..571b4770 100644 --- a/src/patch.rs +++ b/src/patch.rs @@ -1,7 +1,9 @@ //! Code for applying patches to model input files. use anyhow::{Context, Result, ensure}; use csv::{ReaderBuilder, Trim, Writer}; -use indexmap::IndexSet; +use indexmap::{IndexSet, indexset}; +use map_macro::hash_map; +use std::collections::HashMap; use std::fs; use std::path::{Path, PathBuf}; @@ -108,17 +110,88 @@ impl ModelPatch { } } +/// Represents all rows and columns of a CSV file. +/// +/// Assumes that each row is unique (as it should be for all MUSE2 input files). +type CSVTable = IndexSet>; + +/// The replacements to carry out for a file patch. +#[derive(Clone, strum::Display)] +enum Replacements { + /// Delete or add matching rows + Rows { + to_delete: CSVTable, + to_add: CSVTable, + }, + /// Find and replace single values + Values { to_replace: HashMap }, +} + +fn replace_rows(rows: &mut CSVTable, to_add: &CSVTable, to_delete: &CSVTable) -> Result<()> { + // Check that there's no overlap between additions and deletions + for del_row in to_delete { + ensure!( + !to_add.contains(del_row), + "Row appears in both deletions and additions: {del_row:?}", + ); + } + + // Ensure every row requested for deletion actually exists in the base file. + for del_row in to_delete { + ensure!( + rows.contains(del_row), + "Row to delete not present in base file: {del_row:?}" + ); + } + + // Apply deletions + rows.retain(|row| !to_delete.contains(row)); + + // Apply additions (append to end, checking for duplicates) + for add_row in to_add { + ensure!( + rows.insert(add_row.clone()), + "Addition already present in base file: {add_row:?}" + ); + } + + Ok(()) +} + +impl Replacements { + fn apply(&self, rows: &mut CSVTable) -> Result<()> { + match self { + Self::Rows { to_delete, to_add } => replace_rows(rows, to_add, to_delete)?, + Self::Values { to_replace } => { + let mut to_add = CSVTable::new(); + let mut to_delete = CSVTable::new(); + for row in rows.iter() { + if row.iter().any(|value| to_replace.contains_key(value)) { + to_delete.insert(row.clone()); + let new_row = row + .iter() + .map(|old_value| to_replace.get(old_value).unwrap_or(old_value).clone()) + .collect(); + to_add.insert(new_row); + } + } + replace_rows(rows, &to_add, &to_delete)?; + } + } + + Ok(()) + } +} + /// Structure to hold patches for a model csv file. -#[derive(Debug, Clone)] +#[derive(Clone)] pub struct FilePatch { /// The file that this patch applies to (e.g. "agents.csv") filename: String, /// The header row (optional). If `None`, the header is not checked against base files. header_row: Option>, - /// Rows to delete (each row is a vector of fields) - to_delete: IndexSet>, - /// Rows to add (each row is a vector of fields) - to_add: IndexSet>, + /// The replacement(s) to carry out + replacements: Option, } impl FilePatch { @@ -127,8 +200,7 @@ impl FilePatch { FilePatch { filename: filename.into(), header_row: None, - to_delete: IndexSet::new(), - to_add: IndexSet::new(), + replacements: None, } } @@ -148,7 +220,21 @@ impl FilePatch { pub fn with_addition(mut self, row: impl Into) -> Self { let s = row.into(); let v = s.split(',').map(|s| s.trim().to_string()).collect(); - self.to_add.insert(v); + + match self.replacements { + None => { + self.replacements = Some(Replacements::Rows { + to_delete: indexset![], + to_add: indexset![v], + }); + } + Some(Replacements::Rows { + to_delete: _, + ref mut to_add, + }) => assert!(to_add.insert(v), "Attempted to add duplicate row: {s:?}"), + Some(r) => panic!("Cannot add rows when replacement type is already set to: {r}"), + } + self } @@ -156,7 +242,48 @@ impl FilePatch { pub fn with_deletion(mut self, row: impl Into) -> Self { let s = row.into(); let v = s.split(',').map(|s| s.trim().to_string()).collect(); - self.to_delete.insert(v); + + match self.replacements { + None => { + self.replacements = Some(Replacements::Rows { + to_delete: indexset![v], + to_add: indexset![], + }); + } + Some(Replacements::Rows { + ref mut to_delete, + to_add: _, + }) => assert!( + to_delete.insert(v), + "Attempted to delete duplicate row: {s:?}" + ), + Some(r) => panic!("Cannot delete rows when replacement type is already set to: {r}"), + } + + self + } + + /// Replace a value if found in any field + pub fn with_replace_value( + mut self, + old_value: impl Into, + new_value: impl Into, + ) -> Self { + let old_value = old_value.into(); + let new_value = new_value.into(); + match self.replacements { + None => { + self.replacements = Some(Replacements::Values { + to_replace: hash_map! { old_value => new_value }, + }); + } + Some(Replacements::Values { ref mut to_replace }) => assert!( + to_replace.insert(old_value, new_value).is_none(), + "Attempted to replace same value multiple times" + ), + Some(r) => panic!("Cannot replace values when replacement type is already set to: {r}"), + } + self } @@ -229,7 +356,7 @@ fn modify_base_with_patch(base: &str, patch: &FilePatch) -> Result { } // Read all rows from the base, preserving order and checking for duplicates - let mut base_rows: IndexSet> = IndexSet::new(); + let mut base_rows: CSVTable = CSVTable::new(); for result in reader.records() { let record = result?; @@ -246,31 +373,8 @@ fn modify_base_with_patch(base: &str, patch: &FilePatch) -> Result { ); } - // Check that there's no overlap between additions and deletions - for del_row in &patch.to_delete { - ensure!( - !patch.to_add.contains(del_row), - "Row appears in both deletions and additions: {del_row:?}", - ); - } - - // Ensure every row requested for deletion actually exists in the base file. - for del_row in &patch.to_delete { - ensure!( - base_rows.contains(del_row), - "Row to delete not present in base file: {del_row:?}" - ); - } - - // Apply deletions - base_rows.retain(|row| !patch.to_delete.contains(row)); - - // Apply additions (append to end, checking for duplicates) - for add_row in &patch.to_add { - ensure!( - base_rows.insert(add_row.clone()), - "Addition already present in base file: {add_row:?}" - ); + if let Some(replacements) = &patch.replacements { + replacements.apply(&mut base_rows)?; } // Serialize CSV output using csv::Writer @@ -355,7 +459,7 @@ mod tests { } #[test] - fn file_patch() { + fn file_patch_row() { // Patch with a small change to an asset capacity let assets_patch = FilePatch::new("assets.csv") .with_deletion("GASDRV,GBR,A0_GEX,4002.26,2020") @@ -374,6 +478,23 @@ mod tests { assert!(assets_content.contains("GASDRV,GBR,A0_GEX,4003.26,2020")); } + #[test] + fn file_patch_value() { + // Replace decision rule with a different (made-up) one + let agents_patch = FilePatch::new("agents.csv").with_replace_value("single", "madeup"); + + // Build patched model into a temporary directory + let model_dir = ModelPatch::from_example("simple") + .with_file_patch(agents_patch) + .build_to_tempdir() + .unwrap(); + + let agents_path = model_dir.path().join("agents.csv"); + let agents_content = std::fs::read_to_string(agents_path).unwrap(); + assert!(!agents_content.contains("single")); + assert!(agents_content.contains("madeup")); + } + #[test] fn toml_patch() { // Patch to add an extra milestone year (2050) diff --git a/tests/regression.rs b/tests/regression.rs index e4dd53b6..b60ac228 100644 --- a/tests/regression.rs +++ b/tests/regression.rs @@ -25,6 +25,7 @@ define_regression_test!(circularity); // Patched examples define_regression_test_with_patches!(simple_divisible); +define_regression_test_with_patches!(simple_npv); // ------ END: regression tests ------