From bff70eb5550c92ceb842913fa52aefaa5d038211 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 6 Apr 2026 18:52:19 +0000 Subject: [PATCH 1/3] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.14.10 → v0.15.9](https://github.com/astral-sh/ruff-pre-commit/compare/v0.14.10...v0.15.9) - [github.com/codespell-project/codespell: v2.4.1 → v2.4.2](https://github.com/codespell-project/codespell/compare/v2.4.1...v2.4.2) - [github.com/macisamuele/language-formatters-pre-commit-hooks: v2.15.0 → v2.16.0](https://github.com/macisamuele/language-formatters-pre-commit-hooks/compare/v2.15.0...v2.16.0) - [github.com/snakemake/snakefmt: v0.11.2 → v1.0.0](https://github.com/snakemake/snakefmt/compare/v0.11.2...v1.0.0) --- .pre-commit-config.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2600cfed0..761daa77a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,7 +16,7 @@ repos: # Run ruff to lint and format - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.14.10 + rev: v0.15.9 hooks: # Run the linter. - id: ruff @@ -26,7 +26,7 @@ repos: # Find common spelling mistakes in comments and docstrings - repo: https://github.com/codespell-project/codespell - rev: v2.4.1 + rev: v2.4.2 hooks: - id: codespell args: ['--ignore-regex="(\b[A-Z]+\b)"', '--ignore-words-list=fom,appartment,bage,ore,setis,tabacco,berfore,vor,pris,bund,ons,GuD,Dezember,Juni,Juli,Produktion,WorstCase'] # Ignore capital case words, e.g. country codes @@ -42,7 +42,7 @@ repos: # YAML formatting - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks - rev: v2.15.0 + rev: v2.16.0 hooks: - id: pretty-format-yaml exclude: (\.lock\.yaml)$ @@ -50,6 +50,6 @@ repos: # Format Snakemake rule / workflow files - repo: https://github.com/snakemake/snakefmt - rev: v0.11.2 + rev: v1.0.0 hooks: - id: snakefmt From 973cc150ee9e092be009ab59e3370edabbcb89c4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 6 Apr 2026 18:54:16 +0000 Subject: [PATCH 2/3] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- Snakefile | 18 +- rules/build_electricity.smk | 392 ++++---- rules/build_sector.smk | 882 +++++++++--------- rules/collect.smk | 32 +- rules/development.smk | 72 +- rules/postprocess.smk | 294 +++--- rules/pypsa-de/modifications.smk | 180 ++-- rules/pypsa-de/reporting.smk | 60 +- rules/pypsa-de/retrieve.smk | 24 +- rules/retrieve.smk | 348 +++---- rules/solve_electricity.smk | 42 +- rules/solve_myopic.smk | 110 +-- rules/solve_overnight.smk | 30 +- rules/solve_perfect.smk | 90 +- scripts/build_biomass_potentials.py | 8 +- scripts/build_gas_network.py | 6 +- scripts/build_retro_cost.py | 8 +- scripts/build_tyndp_network.py | 12 +- scripts/clean_osm_data.py | 12 +- scripts/cluster_gas_network.py | 6 +- scripts/make_network_comparison.py | 8 +- scripts/plot_hydrogen_network.py | 8 +- .../pypsa-de/build_wasserstoff_kernnetz.py | 12 +- .../pypsa-de/cluster_wasserstoff_kernnetz.py | 11 +- scripts/pypsa-de/plot_ariadne_report.py | 4 +- .../plot_hydrogen_network_incl_kernnetz.py | 12 +- .../prepare_district_heating_subnodes.py | 14 +- scripts/solve_network.py | 6 +- test/test_data_versions_layer.py | 24 +- 29 files changed, 1382 insertions(+), 1343 deletions(-) diff --git a/Snakefile b/Snakefile index b4227fd56..cfff32a55 100644 --- a/Snakefile +++ b/Snakefile @@ -97,6 +97,7 @@ include: "rules/pypsa-de/reporting.smk" rule all: + default_target: True input: expand(RESULTS + "graphs/costs.pdf", run=config["run"]["name"]), expand(resources("maps/power-network.pdf"), run=config["run"]["name"]), @@ -227,7 +228,6 @@ rule all: ), lambda w: balance_map_paths("static", w), lambda w: balance_map_paths("interactive", w), - default_target: True rule create_scenarios: @@ -277,9 +277,6 @@ rule dump_graph_config: rule rulegraph: - """Generates Rule DAG in DOT, PDF, PNG, and SVG formats using the final configuration.""" - message: - "Creating RULEGRAPH dag in multiple formats using the final configuration." input: config_file=rules.dump_graph_config.output.config_file, output: @@ -287,6 +284,9 @@ rule rulegraph: pdf=resources("dag_rulegraph.pdf"), png=resources("dag_rulegraph.png"), svg=resources("dag_rulegraph.svg"), + """Generates Rule DAG in DOT, PDF, PNG, and SVG formats using the final configuration.""" + message: + "Creating RULEGRAPH dag in multiple formats using the final configuration." shell: r""" # Generate DOT file using nested snakemake with the dumped final config @@ -314,9 +314,6 @@ rule rulegraph: rule filegraph: - """Generates File DAG in DOT, PDF, PNG, and SVG formats using the final configuration.""" - message: - "Creating FILEGRAPH dag in multiple formats using the final configuration." input: config_file=rules.dump_graph_config.output.config_file, output: @@ -324,6 +321,9 @@ rule filegraph: pdf=resources("dag_filegraph.pdf"), png=resources("dag_filegraph.png"), svg=resources("dag_filegraph.svg"), + """Generates File DAG in DOT, PDF, PNG, and SVG formats using the final configuration.""" + message: + "Creating FILEGRAPH dag in multiple formats using the final configuration." shell: r""" # Generate DOT file using nested snakemake with the dumped final config @@ -350,10 +350,10 @@ rule filegraph: rule doc: - message: - "Build documentation." output: directory("doc/_build"), + message: + "Build documentation." shell: "pixi run build-docs {output} html" diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 495423b03..d8d1fe5c8 100755 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -4,13 +4,6 @@ rule build_electricity_demand: - message: - "Building electricity demand time series" - params: - snapshots=config_provider("snapshots"), - drop_leap_day=config_provider("enable", "drop_leap_day"), - countries=config_provider("countries"), - load=config_provider("load"), input: opsd=rules.retrieve_electricity_demand_opsd.output["csv"], neso=rules.retrieve_electricity_demand_neso.output["csv"], @@ -28,18 +21,18 @@ rule build_electricity_demand: benchmarks("build_electricity_demand") resources: mem_mb=5000, + params: + snapshots=config_provider("snapshots"), + drop_leap_day=config_provider("enable", "drop_leap_day"), + countries=config_provider("countries"), + load=config_provider("load"), + message: + "Building electricity demand time series" script: scripts("build_electricity_demand.py") rule build_powerplants: - message: - "Building powerplant list for {wildcards.clusters} clusters" - params: - powerplants_filter=config_provider("electricity", "powerplants_filter"), - custom_powerplants=config_provider("electricity", "custom_powerplants"), - everywhere_powerplants=config_provider("electricity", "everywhere_powerplants"), - countries=config_provider("countries"), input: network=resources("networks/base_s_{clusters}.nc"), regions_onshore=resources("regions_onshore_base_s_{clusters}.geojson"), @@ -55,6 +48,13 @@ rule build_powerplants: threads: 1 resources: mem_mb=7000, + params: + powerplants_filter=config_provider("electricity", "powerplants_filter"), + custom_powerplants=config_provider("electricity", "custom_powerplants"), + everywhere_powerplants=config_provider("electricity", "everywhere_powerplants"), + countries=config_provider("countries"), + message: + "Building powerplant list for {wildcards.clusters} clusters" script: scripts("build_powerplants.py") @@ -78,17 +78,6 @@ def input_base_network(w): rule base_network: - message: - "Building base network" - params: - countries=config_provider("countries"), - snapshots=config_provider("snapshots"), - drop_leap_day=config_provider("enable", "drop_leap_day"), - lines=config_provider("lines"), - links=config_provider("links"), - transformers=config_provider("transformers"), - clustering=config_provider("clustering", "mode"), - admin_levels=config_provider("clustering", "administrative"), input: unpack(input_base_network), nuts3_shapes=resources("nuts3_shapes.geojson"), @@ -107,13 +96,22 @@ rule base_network: threads: 4 resources: mem_mb=2000, + params: + countries=config_provider("countries"), + snapshots=config_provider("snapshots"), + drop_leap_day=config_provider("enable", "drop_leap_day"), + lines=config_provider("lines"), + links=config_provider("links"), + transformers=config_provider("transformers"), + clustering=config_provider("clustering", "mode"), + admin_levels=config_provider("clustering", "administrative"), + message: + "Building base network" script: scripts("base_network.py") rule build_osm_boundaries: - message: - "Building OSM boundaries for {wildcards.country}" input: json=f"{OSM_BOUNDARIES_DATASET['folder']}/{{country}}_adm1.json", eez=ancient(rules.retrieve_eez.output["gpkg"]), @@ -124,21 +122,13 @@ rule build_osm_boundaries: threads: 1 resources: mem_mb=1500, + message: + "Building OSM boundaries for {wildcards.country}" script: scripts("build_osm_boundaries.py") rule build_bidding_zones: - message: - "Building bidding zones" - params: - countries=config_provider("countries"), - remove_islands=config_provider( - "clustering", "build_bidding_zones", "remove_islands" - ), - aggregate_to_tyndp=config_provider( - "clustering", "build_bidding_zones", "aggregate_to_tyndp" - ), input: bidding_zones_entsoepy=f"{BIDDING_ZONES_ENTSOEPY_DATASET['folder']}/bidding_zones_entsoepy.geojson", bidding_zones_electricitymaps=f"{BIDDING_ZONES_ELECTRICITYMAPS_DATASET['folder']}/bidding_zones_electricitymaps.geojson", @@ -149,16 +139,21 @@ rule build_bidding_zones: threads: 1 resources: mem_mb=1500, + params: + countries=config_provider("countries"), + remove_islands=config_provider( + "clustering", "build_bidding_zones", "remove_islands" + ), + aggregate_to_tyndp=config_provider( + "clustering", "build_bidding_zones", "aggregate_to_tyndp" + ), + message: + "Building bidding zones" script: scripts("build_bidding_zones.py") rule build_shapes: - message: - "Building geographical shapes" - params: - config_provider("clustering", "mode"), - countries=config_provider("countries"), input: eez=ancient(rules.retrieve_eez.output["gpkg"]), nuts3_2021=rules.retrieve_eu_nuts_2021.output["shapes_level_3"], @@ -187,6 +182,11 @@ rule build_shapes: threads: 1 resources: mem_mb=1500, + params: + config_provider("clustering", "mode"), + countries=config_provider("countries"), + message: + "Building geographical shapes" script: scripts("build_shapes.py") @@ -194,10 +194,6 @@ rule build_shapes: if CUTOUT_DATASET["source"] in ["build"]: rule build_cutout: - message: - "Building cutout data for {wildcards.cutout}" - params: - cutouts=config_provider("atlite", "cutouts"), output: cutout=CUTOUT_DATASET["folder"] / "{cutout}.nc", log: @@ -207,13 +203,15 @@ if CUTOUT_DATASET["source"] in ["build"]: threads: config["atlite"].get("nprocesses", 4) resources: mem_mb=config["atlite"].get("nprocesses", 4) * 1000, + params: + cutouts=config_provider("atlite", "cutouts"), + message: + "Building cutout data for {wildcards.cutout}" script: scripts("build_cutout.py") rule build_ship_raster: - message: - "Building ship density raster" input: ship_density=rules.retrieve_ship_raster.output["zip_file"], cutout=lambda w: input_cutout(w), @@ -221,19 +219,17 @@ rule build_ship_raster: resources("shipdensity_raster.tif"), log: logs("build_ship_raster.log"), - resources: - mem_mb=5000, benchmark: benchmarks("build_ship_raster") + resources: + mem_mb=5000, + message: + "Building ship density raster" script: scripts("build_ship_raster.py") rule determine_availability_matrix_MD_UA: - message: - "Determining availability matrix for {wildcards.clusters} clusters and {wildcards.technology} technology" - params: - renewable=config_provider("renewable"), input: copernicus=rules.download_copernicus_land_cover.output["tif"], wdpa=lambda w: ( @@ -277,6 +273,10 @@ rule determine_availability_matrix_MD_UA: threads: config["atlite"].get("nprocesses", 4) resources: mem_mb=config["atlite"].get("nprocesses", 4) * 5000, + params: + renewable=config_provider("renewable"), + message: + "Determining availability matrix for {wildcards.clusters} clusters and {wildcards.technology} technology" script: scripts("determine_availability_matrix_MD_UA.py") @@ -294,11 +294,6 @@ def input_ua_md_availability_matrix(w): rule determine_availability_matrix: - message: - "Determining availability matrix for {wildcards.clusters} clusters and {wildcards.technology} technology" - params: - renewable=config_provider("renewable"), - plot_availability_matrix=config_provider("atlite", "plot_availability_matrix"), input: unpack(input_ua_md_availability_matrix), corine=ancient(rules.retrieve_corine.output["tif_file"]), @@ -346,17 +341,16 @@ rule determine_availability_matrix: threads: config["atlite"].get("nprocesses", 4) resources: mem_mb=config["atlite"].get("nprocesses", 4) * 5000, + params: + renewable=config_provider("renewable"), + plot_availability_matrix=config_provider("atlite", "plot_availability_matrix"), + message: + "Determining availability matrix for {wildcards.clusters} clusters and {wildcards.technology} technology" script: scripts("determine_availability_matrix.py") rule build_renewable_profiles: - message: - "Building renewable profiles for {wildcards.clusters} clusters and {wildcards.technology} technology" - params: - snapshots=config_provider("snapshots"), - drop_leap_day=config_provider("enable", "drop_leap_day"), - renewable=config_provider("renewable"), input: availability_matrix=resources("availability_matrix_{clusters}_{technology}.nc"), offshore_shapes=resources("offshore_shapes.geojson"), @@ -376,20 +370,22 @@ rule build_renewable_profiles: logs("build_renewable_profile_{clusters}_{technology}.log"), benchmark: benchmarks("build_renewable_profile_{clusters}_{technology}") + wildcard_constraints: + technology="(?!hydro).*", # Any technology other than hydro threads: config["atlite"].get("nprocesses", 4) resources: mem_mb=config["atlite"].get("nprocesses", 4) * 5000, - wildcard_constraints: - technology="(?!hydro).*", # Any technology other than hydro + params: + snapshots=config_provider("snapshots"), + drop_leap_day=config_provider("enable", "drop_leap_day"), + renewable=config_provider("renewable"), + message: + "Building renewable profiles for {wildcards.clusters} clusters and {wildcards.technology} technology" script: scripts("build_renewable_profiles.py") rule build_co2_prices: - message: - "Building CO2 price time series" - params: - rolling_window=config_provider("costs", "emission_prices", "rolling_window"), input: csv=rules.retrieve_co2_prices.output["csv"], output: @@ -401,15 +397,15 @@ rule build_co2_prices: threads: 1 resources: mem_mb=5000, + params: + rolling_window=config_provider("costs", "emission_prices", "rolling_window"), + message: + "Building CO2 price time series" script: "../scripts/build_co2_prices.py" rule build_fossil_fuel_prices: - message: - "Building fossil fuel price time series" - params: - rolling_window=config_provider("conventional", "fuel_price_rolling_window"), input: fuel_price_raw=rules.retrieve_worldbank_commodity_prices.output["xlsx"], output: @@ -421,6 +417,10 @@ rule build_fossil_fuel_prices: threads: 1 resources: mem_mb=5000, + params: + rolling_window=config_provider("conventional", "fuel_price_rolling_window"), + message: + "Building fossil fuel price time series" script: scripts("build_monthly_prices.py") @@ -446,13 +446,6 @@ if COUNTRY_RUNOFF_DATASET["source"] == "build": rule build_hydro_profile: - message: - "Building hydropower profile" - params: - hydro=config_provider("renewable", "hydro"), - countries=config_provider("countries"), - snapshots=config_provider("snapshots"), - drop_leap_day=config_provider("enable", "drop_leap_day"), input: country_shapes=resources("country_shapes.geojson"), eia_hydro_generation="data/eia_hydro_annual_generation.csv", @@ -469,16 +462,18 @@ rule build_hydro_profile: benchmarks("build_hydro_profile") resources: mem_mb=5000, + params: + hydro=config_provider("renewable", "hydro"), + countries=config_provider("countries"), + snapshots=config_provider("snapshots"), + drop_leap_day=config_provider("enable", "drop_leap_day"), + message: + "Building hydropower profile" script: scripts("build_hydro_profile.py") rule build_line_rating: - message: - "Building dynamic line ratings" - params: - snapshots=config_provider("snapshots"), - drop_leap_day=config_provider("enable", "drop_leap_day"), input: base_network=resources("networks/base.nc"), cutout=lambda w: input_cutout( @@ -493,17 +488,16 @@ rule build_line_rating: threads: config["atlite"].get("nprocesses", 4) resources: mem_mb=config["atlite"].get("nprocesses", 4) * 1000, + params: + snapshots=config_provider("snapshots"), + drop_leap_day=config_provider("enable", "drop_leap_day"), + message: + "Building dynamic line ratings" script: scripts("build_line_rating.py") rule build_transmission_projects: - message: - "Building transmission projects" - params: - transmission_projects=config_provider("transmission_projects"), - line_factor=config_provider("lines", "length_factor"), - s_max_pu=config_provider("lines", "s_max_pu"), input: base_network=resources("networks/base.nc"), offshore_shapes=resources("offshore_shapes.geojson"), @@ -525,20 +519,20 @@ rule build_transmission_projects: logs("build_transmission_projects.log"), benchmark: benchmarks("build_transmission_projects") + threads: 1 resources: mem_mb=4000, - threads: 1 + params: + transmission_projects=config_provider("transmission_projects"), + line_factor=config_provider("lines", "length_factor"), + s_max_pu=config_provider("lines", "s_max_pu"), + message: + "Building transmission projects" script: scripts("build_transmission_projects.py") rule add_transmission_projects_and_dlr: - message: - "Adding transmission projects and dynamic line ratings" - params: - transmission_projects=config_provider("transmission_projects"), - dlr=config_provider("lines", "dynamic_line_rating"), - s_max_pu=config_provider("lines", "s_max_pu"), input: network=resources("networks/base.nc"), dlr=lambda w: ( @@ -566,6 +560,12 @@ rule add_transmission_projects_and_dlr: threads: 1 resources: mem_mb=4000, + params: + transmission_projects=config_provider("transmission_projects"), + dlr=config_provider("lines", "dynamic_line_rating"), + s_max_pu=config_provider("lines", "s_max_pu"), + message: + "Adding transmission projects and dynamic line ratings" script: scripts("add_transmission_projects_and_dlr.py") @@ -581,11 +581,6 @@ def input_class_regions(w): rule build_electricity_demand_base: - message: - "Building electricity demand time series for base network" - params: - distribution_key=config_provider("load", "distribution_key"), - substation_only=config_provider("load", "substation_only"), input: base_network=resources("networks/base_s.nc"), regions=resources("regions_onshore_base_s.geojson"), @@ -602,17 +597,16 @@ rule build_electricity_demand_base: benchmarks("build_electricity_demand_base_s") resources: mem_mb=5000, + params: + distribution_key=config_provider("load", "distribution_key"), + substation_only=config_provider("load", "substation_only"), + message: + "Building electricity demand time series for base network" script: scripts("build_electricity_demand_base.py") rule build_hac_features: - message: - "Aggregate all rastered cutout data to base regions Voronoi cells." - params: - snapshots=config_provider("snapshots"), - drop_leap_day=config_provider("enable", "drop_leap_day"), - features=config_provider("clustering", "cluster_network", "hac_features"), input: cutout=lambda w: input_cutout(w), regions=resources("regions_onshore_base_s.geojson"), @@ -625,14 +619,17 @@ rule build_hac_features: threads: config["atlite"].get("nprocesses", 4) resources: mem_mb=10000, + params: + snapshots=config_provider("snapshots"), + drop_leap_day=config_provider("enable", "drop_leap_day"), + features=config_provider("clustering", "cluster_network", "hac_features"), + message: + "Aggregate all rastered cutout data to base regions Voronoi cells." script: scripts("build_hac_features.py") rule process_cost_data: - params: - costs=config_provider("costs"), - max_hours=config_provider("electricity", "max_hours"), input: network=resources("networks/base_s.nc"), costs=rules.retrieve_cost_data.output["costs"], @@ -646,24 +643,14 @@ rule process_cost_data: threads: 1 resources: mem_mb=4000, + params: + costs=config_provider("costs"), + max_hours=config_provider("electricity", "max_hours"), script: scripts("process_cost_data.py") rule simplify_network: - message: - "Simplifying network" - params: - countries=config_provider("countries"), - mode=config_provider("clustering", "mode"), - administrative=config_provider("clustering", "administrative"), - simplify_network=config_provider("clustering", "simplify_network"), - cluster_network=config_provider("clustering", "cluster_network"), - aggregation_strategies=config_provider( - "clustering", "aggregation_strategies", default={} - ), - p_max_pu=config_provider("links", "p_max_pu", default=1.0), - p_min_pu=config_provider("links", "p_min_pu", default=-1.0), input: network=resources("networks/base_extended.nc"), regions_onshore=resources("regions_onshore.geojson"), @@ -681,6 +668,19 @@ rule simplify_network: threads: 1 resources: mem_mb=24000, + params: + countries=config_provider("countries"), + mode=config_provider("clustering", "mode"), + administrative=config_provider("clustering", "administrative"), + simplify_network=config_provider("clustering", "simplify_network"), + cluster_network=config_provider("clustering", "cluster_network"), + aggregation_strategies=config_provider( + "clustering", "aggregation_strategies", default={} + ), + p_max_pu=config_provider("links", "p_max_pu", default=1.0), + p_min_pu=config_provider("links", "p_min_pu", default=-1.0), + message: + "Simplifying network" script: scripts("simplify_network.py") @@ -708,25 +708,6 @@ def input_custom_busmap(w): rule cluster_network: - message: - "Clustering network to {wildcards.clusters} clusters" - params: - countries=config_provider("countries"), - mode=config_provider("clustering", "mode"), - administrative=config_provider("clustering", "administrative"), - cluster_network=config_provider("clustering", "cluster_network"), - aggregation_strategies=config_provider( - "clustering", "aggregation_strategies", default={} - ), - focus_weights=config_provider("clustering", "focus_weights", default=None), - renewable_carriers=config_provider("electricity", "renewable_carriers"), - conventional_carriers=config_provider( - "electricity", "conventional_carriers", default=[] - ), - max_hours=config_provider("electricity", "max_hours"), - length_factor=config_provider("lines", "length_factor"), - cluster_mode=config_provider("clustering", "mode"), - copperplate_regions=config_provider("clustering", "copperplate_regions"), input: unpack(input_custom_busmap), network=resources("networks/base_s.nc"), @@ -758,6 +739,25 @@ rule cluster_network: threads: 1 resources: mem_mb=10000, + params: + countries=config_provider("countries"), + mode=config_provider("clustering", "mode"), + administrative=config_provider("clustering", "administrative"), + cluster_network=config_provider("clustering", "cluster_network"), + aggregation_strategies=config_provider( + "clustering", "aggregation_strategies", default={} + ), + focus_weights=config_provider("clustering", "focus_weights", default=None), + renewable_carriers=config_provider("electricity", "renewable_carriers"), + conventional_carriers=config_provider( + "electricity", "conventional_carriers", default=[] + ), + max_hours=config_provider("electricity", "max_hours"), + length_factor=config_provider("lines", "length_factor"), + cluster_mode=config_provider("clustering", "mode"), + copperplate_regions=config_provider("clustering", "copperplate_regions"), + message: + "Clustering network to {wildcards.clusters} clusters" script: scripts("cluster_network.py") @@ -788,24 +788,6 @@ def input_conventional(w): rule add_electricity: - message: - "Adding electricity to network with {wildcards.clusters} clusters" - params: - line_length_factor=config_provider("lines", "length_factor"), - link_length_factor=config_provider("links", "length_factor"), - scaling_factor=config_provider("load", "scaling_factor"), - countries=config_provider("countries"), - snapshots=config_provider("snapshots"), - renewable=config_provider("renewable"), - electricity=config_provider("electricity"), - conventional=config_provider("conventional"), - foresight=config_provider("foresight"), - drop_leap_day=config_provider("enable", "drop_leap_day"), - consider_efficiency_classes=config_provider( - "clustering", "consider_efficiency_classes" - ), - aggregation_strategies=config_provider("clustering", "aggregation_strategies"), - exclude_carriers=config_provider("clustering", "exclude_carriers"), input: unpack(input_profile_tech), unpack(input_class_regions), @@ -834,27 +816,29 @@ rule add_electricity: threads: 1 resources: mem_mb=10000, + params: + line_length_factor=config_provider("lines", "length_factor"), + link_length_factor=config_provider("links", "length_factor"), + scaling_factor=config_provider("load", "scaling_factor"), + countries=config_provider("countries"), + snapshots=config_provider("snapshots"), + renewable=config_provider("renewable"), + electricity=config_provider("electricity"), + conventional=config_provider("conventional"), + foresight=config_provider("foresight"), + drop_leap_day=config_provider("enable", "drop_leap_day"), + consider_efficiency_classes=config_provider( + "clustering", "consider_efficiency_classes" + ), + aggregation_strategies=config_provider("clustering", "aggregation_strategies"), + exclude_carriers=config_provider("clustering", "exclude_carriers"), + message: + "Adding electricity to network with {wildcards.clusters} clusters" script: scripts("add_electricity.py") rule prepare_network: - message: - "Preparing network for model with {wildcards.clusters} clusters and options {wildcards.opts}" - params: - time_resolution=config_provider("clustering", "temporal", "resolution_elec"), - links=config_provider("links"), - lines=config_provider("lines"), - co2base=config_provider("electricity", "co2base"), - co2limit_enable=config_provider("electricity", "co2limit_enable", default=False), - co2limit=config_provider("electricity", "co2limit"), - gaslimit_enable=config_provider("electricity", "gaslimit_enable", default=False), - gaslimit=config_provider("electricity", "gaslimit"), - emission_prices=config_provider("costs", "emission_prices"), - adjustments=config_provider("adjustments", "electricity"), - autarky=config_provider("electricity", "autarky", default={}), - drop_leap_day=config_provider("enable", "drop_leap_day"), - transmission_limit=config_provider("electricity", "transmission_limit"), input: resources("networks/base_s_{clusters}_elec.nc"), costs=lambda w: resources( @@ -874,15 +858,27 @@ rule prepare_network: threads: 1 resources: mem_mb=4000, + params: + time_resolution=config_provider("clustering", "temporal", "resolution_elec"), + links=config_provider("links"), + lines=config_provider("lines"), + co2base=config_provider("electricity", "co2base"), + co2limit_enable=config_provider("electricity", "co2limit_enable", default=False), + co2limit=config_provider("electricity", "co2limit"), + gaslimit_enable=config_provider("electricity", "gaslimit_enable", default=False), + gaslimit=config_provider("electricity", "gaslimit"), + emission_prices=config_provider("costs", "emission_prices"), + adjustments=config_provider("adjustments", "electricity"), + autarky=config_provider("electricity", "autarky", default={}), + drop_leap_day=config_provider("enable", "drop_leap_day"), + transmission_limit=config_provider("electricity", "transmission_limit"), + message: + "Preparing network for model with {wildcards.clusters} clusters and options {wildcards.opts}" script: scripts("prepare_network.py") rule clean_osm_data: - message: - "Cleaning raw OSM data for countries: " + ", ".join(config["countries"]) - params: - voltages=config_provider("electricity", "voltages"), input: cables_way=expand( f"{OSM_DATASET['folder']}/{{country}}/cables_way.json", @@ -921,19 +917,15 @@ rule clean_osm_data: threads: 1 resources: mem_mb=4000, + params: + voltages=config_provider("electricity", "voltages"), + message: + "Cleaning raw OSM data for countries: " + ", ".join(config["countries"]) script: "../scripts/clean_osm_data.py" rule build_osm_network: - message: - "Building OSM network" - params: - countries=config_provider("countries"), - voltages=config_provider("electricity", "voltages"), - line_types=config_provider("lines", "types"), - under_construction=config_provider("osm_network_release", "under_construction"), - remove_after=config_provider("osm_network_release", "remove_after"), input: substations=resources(f"osm/clean/substations.geojson"), substations_polygon=resources(f"osm/clean/substations_polygon.geojson"), @@ -963,15 +955,19 @@ rule build_osm_network: threads: 1 resources: mem_mb=4000, + params: + countries=config_provider("countries"), + voltages=config_provider("electricity", "voltages"), + line_types=config_provider("lines", "types"), + under_construction=config_provider("osm_network_release", "under_construction"), + remove_after=config_provider("osm_network_release", "remove_after"), + message: + "Building OSM network" script: "../scripts/build_osm_network.py" rule build_tyndp_network: - message: - "Building TYNDP network" - params: - countries=config_provider("countries"), input: reference_grid=rules.retrieve_tyndp.output.reference_grid, buses=rules.retrieve_tyndp.output.nodes, @@ -996,5 +992,9 @@ rule build_tyndp_network: threads: 1 resources: mem_mb=4000, + params: + countries=config_provider("countries"), + message: + "Building TYNDP network" script: "../scripts/build_tyndp_network.py" diff --git a/rules/build_sector.smk b/rules/build_sector.smk index d4bf10a1f..89ee57920 100755 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -4,8 +4,6 @@ rule build_population_layouts: - message: - "Building population layout data (total, urban, rural) from NUTS3 shapes and World Bank statistics" input: nuts3_shapes=resources("nuts3_shapes.geojson"), urban_percent=rules.retrieve_worldbank_urban_population.output["csv"], @@ -16,18 +14,18 @@ rule build_population_layouts: pop_layout_rural=resources("pop_layout_rural.nc"), log: logs("build_population_layouts.log"), - resources: - mem_mb=20000, benchmark: benchmarks("build_population_layouts") threads: 8 + resources: + mem_mb=20000, + message: + "Building population layout data (total, urban, rural) from NUTS3 shapes and World Bank statistics" script: scripts("build_population_layouts.py") rule build_clustered_population_layouts: - message: - "Clustering population layouts for {wildcards.clusters} clusters" input: pop_layout_total=resources("pop_layout_total.nc"), pop_layout_urban=resources("pop_layout_urban.nc"), @@ -38,17 +36,17 @@ rule build_clustered_population_layouts: clustered_pop_layout=resources("pop_layout_base_s_{clusters}.csv"), log: logs("build_clustered_population_layouts_s_{clusters}.log"), - resources: - mem_mb=10000, benchmark: benchmarks("build_clustered_population_layouts/s_{clusters}") + resources: + mem_mb=10000, + message: + "Clustering population layouts for {wildcards.clusters} clusters" script: scripts("build_clustered_population_layouts.py") rule build_clustered_solar_rooftop_potentials: - message: - "Building solar rooftop potentials for {wildcards.clusters} clusters" input: pop_layout=resources("pop_layout_total.nc"), class_regions=resources("regions_by_class_{clusters}_solar.geojson"), @@ -57,17 +55,17 @@ rule build_clustered_solar_rooftop_potentials: potentials=resources("solar_rooftop_potentials_s_{clusters}.csv"), log: logs("build_clustered_solar_rooftop_potentials_s_{clusters}.log"), - resources: - mem_mb=10000, benchmark: benchmarks("build_clustered_solar_rooftop_potentials/s_{clusters}") + resources: + mem_mb=10000, + message: + "Building solar rooftop potentials for {wildcards.clusters} clusters" script: scripts("build_clustered_solar_rooftop_potentials.py") rule build_simplified_population_layouts: - message: - "Building simplified population layouts for base scenario" input: pop_layout_total=resources("pop_layout_total.nc"), pop_layout_urban=resources("pop_layout_urban.nc"), @@ -76,36 +74,36 @@ rule build_simplified_population_layouts: cutout=lambda w: input_cutout(w), output: clustered_pop_layout=resources("pop_layout_base_s.csv"), - resources: - mem_mb=10000, log: logs("build_simplified_population_layouts_s"), benchmark: benchmarks("build_simplified_population_layouts/s") + resources: + mem_mb=10000, + message: + "Building simplified population layouts for base scenario" script: scripts("build_clustered_population_layouts.py") rule build_gas_network: - message: - "Building cleaned gas network from SciGRID-Gas data" input: gas_network=rules.retrieve_gas_infrastructure_data.output["gas_network"], output: cleaned_gas_network=resources("gas_network.csv"), - resources: - mem_mb=4000, log: logs("build_gas_network.log"), benchmark: benchmarks("build_gas_network") + resources: + mem_mb=4000, + message: + "Building cleaned gas network from SciGRID-Gas data" script: scripts("build_gas_network.py") rule build_gas_input_locations: - message: - "Building gas input locations for {wildcards.clusters} clusters" input: gem="data/gem/Europe-Gas-Tracker-2024-05.xlsx", entry=rules.retrieve_gas_infrastructure_data.output["entry"], @@ -117,41 +115,38 @@ rule build_gas_input_locations: gas_input_nodes_simplified=resources( "gas_input_locations_s_{clusters}_simplified.csv" ), - resources: - mem_mb=2000, log: logs("build_gas_input_locations_s_{clusters}.log"), benchmark: benchmarks("build_gas_input_locations/s_{clusters}") + resources: + mem_mb=2000, + message: + "Building gas input locations for {wildcards.clusters} clusters" script: scripts("build_gas_input_locations.py") rule cluster_gas_network: - message: - "Clustering gas network for {wildcards.clusters} clusters" input: cleaned_gas_network=resources("gas_network.csv"), regions_onshore=resources("regions_onshore_base_s_{clusters}.geojson"), regions_offshore=resources("regions_offshore_base_s_{clusters}.geojson"), output: clustered_gas_network=resources("gas_network_base_s_{clusters}.csv"), - resources: - mem_mb=4000, log: logs("cluster_gas_network_{clusters}.log"), benchmark: benchmarks("cluster_gas_network/s_{clusters}") + resources: + mem_mb=4000, + message: + "Clustering gas network for {wildcards.clusters} clusters" script: scripts("cluster_gas_network.py") rule build_daily_heat_demand: - message: - "Building daily heat demand profiles for {wildcards.clusters} clusters" - params: - snapshots=config_provider("snapshots"), - drop_leap_day=config_provider("enable", "drop_leap_day"), input: pop_layout=resources("pop_layout_total.nc"), regions_onshore=resources("regions_onshore_base_s_{clusters}.geojson"), @@ -160,24 +155,23 @@ rule build_daily_heat_demand: ), output: heat_demand=resources("daily_heat_demand_total_base_s_{clusters}.nc"), - resources: - mem_mb=20000, - threads: 8 log: logs("build_daily_heat_demand_total_s_{clusters}.loc"), benchmark: benchmarks("build_daily_heat_demand/total_s_{clusters}") + threads: 8 + resources: + mem_mb=20000, + params: + snapshots=config_provider("snapshots"), + drop_leap_day=config_provider("enable", "drop_leap_day"), + message: + "Building daily heat demand profiles for {wildcards.clusters} clusters" script: scripts("build_daily_heat_demand.py") rule build_hourly_heat_demand: - message: - "Building hourly heat demand profiles from daily demand for {wildcards.clusters} clusters" - params: - snapshots=config_provider("snapshots"), - drop_leap_day=config_provider("enable", "drop_leap_day"), - sector=config_provider("sector"), input: heat_profile="data/heat_load_profile_BDEW.csv", heat_demand=resources("daily_heat_demand_total_base_s_{clusters}.nc"), @@ -186,23 +180,24 @@ rule build_hourly_heat_demand: heat_dsm_profile=resources( "residential_heat_dsm_profile_total_base_s_{clusters}.csv" ), - resources: - mem_mb=2000, - threads: 8 log: logs("build_hourly_heat_demand_total_s_{clusters}.loc"), benchmark: benchmarks("build_hourly_heat_demand/total_s_{clusters}") + threads: 8 + resources: + mem_mb=2000, + params: + snapshots=config_provider("snapshots"), + drop_leap_day=config_provider("enable", "drop_leap_day"), + sector=config_provider("sector"), + message: + "Building hourly heat demand profiles from daily demand for {wildcards.clusters} clusters" script: scripts("build_hourly_heat_demand.py") rule build_temperature_profiles: - message: - "Building temperature profiles for {wildcards.clusters} clusters" - params: - snapshots=config_provider("snapshots"), - drop_leap_day=config_provider("enable", "drop_leap_day"), input: pop_layout=resources("pop_layout_total.nc"), regions_onshore=lambda w: ( @@ -216,20 +211,47 @@ rule build_temperature_profiles: output: temp_soil=resources("temp_soil_total_base_s_{clusters}.nc"), temp_air=resources("temp_air_total_base_s_{clusters}.nc"), - resources: - mem_mb=20000, - threads: 8 log: logs("build_temperature_profiles_total_s_{clusters}.log"), benchmark: benchmarks("build_temperature_profiles/total_{clusters}") + threads: 8 + resources: + mem_mb=20000, + params: + snapshots=config_provider("snapshots"), + drop_leap_day=config_provider("enable", "drop_leap_day"), + message: + "Building temperature profiles for {wildcards.clusters} clusters" script: scripts("build_temperature_profiles.py") rule build_central_heating_temperature_profiles: - message: - "Building central heating temperature profiles for {wildcards.clusters} clusters and {wildcards.planning_horizons} planning horizon" + input: + temp_air_total=resources("temp_air_total_base_s_{clusters}.nc"), + regions_onshore=lambda w: ( + resources("regions_onshore_base-extended_s_{clusters}.geojson") + if config_provider("sector", "district_heating", "subnodes", "enable")(w) + else resources("regions_onshore_base_s_{clusters}.geojson") + ), + output: + central_heating_forward_temperature_profiles=resources( + "central_heating_forward_temperature_profiles_base_s_{clusters}_{planning_horizons}.nc" + ), + central_heating_return_temperature_profiles=resources( + "central_heating_return_temperature_profiles_base_s_{clusters}_{planning_horizons}.nc" + ), + log: + logs( + "build_central_heating_temperature_profiles_s_{clusters}_{planning_horizons}.log" + ), + benchmark: + benchmarks( + "build_central_heating_temperature_profiles/s_{clusters}_{planning_horizons}" + ) + resources: + mem_mb=20000, params: max_forward_temperature_central_heating_baseyear=config_provider( "sector", @@ -276,75 +298,34 @@ rule build_central_heating_temperature_profiles: "relative_annual_temperature_reduction", ), energy_totals_year=config_provider("energy", "energy_totals_year"), - input: - temp_air_total=resources("temp_air_total_base_s_{clusters}.nc"), - regions_onshore=lambda w: ( - resources("regions_onshore_base-extended_s_{clusters}.geojson") - if config_provider("sector", "district_heating", "subnodes", "enable")(w) - else resources("regions_onshore_base_s_{clusters}.geojson") - ), - output: - central_heating_forward_temperature_profiles=resources( - "central_heating_forward_temperature_profiles_base_s_{clusters}_{planning_horizons}.nc" - ), - central_heating_return_temperature_profiles=resources( - "central_heating_return_temperature_profiles_base_s_{clusters}_{planning_horizons}.nc" - ), - resources: - mem_mb=20000, - log: - logs( - "build_central_heating_temperature_profiles_s_{clusters}_{planning_horizons}.log" - ), - benchmark: - benchmarks( - "build_central_heating_temperature_profiles/s_{clusters}_{planning_horizons}" - ) + message: + "Building central heating temperature profiles for {wildcards.clusters} clusters and {wildcards.planning_horizons} planning horizon" script: scripts("build_central_heating_temperature_profiles/run.py") rule build_dh_areas: - params: - handle_missing_countries=config_provider( - "sector", "district_heating", "dh_areas", "handle_missing_countries" - ), - countries=config_provider("countries"), input: dh_areas=rules.retrieve_dh_areas.output["dh_areas"], regions_onshore=resources("regions_onshore_base_s_{clusters}.geojson"), output: dh_areas=resources("dh_areas_base_s_{clusters}.geojson"), - resources: - mem_mb=2000, log: logs("build_dh_areas_s_{clusters}.log"), benchmark: benchmarks("build_dh_areas_s/s_{clusters}") + resources: + mem_mb=2000, + params: + handle_missing_countries=config_provider( + "sector", "district_heating", "dh_areas", "handle_missing_countries" + ), + countries=config_provider("countries"), script: scripts("build_dh_areas.py") rule build_geothermal_heat_potential: - message: - "Building geothermal heat potential estimates for {wildcards.clusters} clusters" - params: - drop_leap_day=config_provider("enable", "drop_leap_day"), - countries=config_provider("countries"), - constant_temperature_celsius=config_provider( - "sector", - "district_heating", - "limited_heat_sources", - "geothermal", - "constant_temperature_celsius", - ), - ignore_missing_regions=config_provider( - "sector", - "district_heating", - "limited_heat_sources", - "geothermal", - "ignore_missing_regions", - ), input: isi_heat_potentials=rules.retrieve_geothermal_heat_utilisation_potentials.output[ "isi_heat_potentials" @@ -359,19 +340,56 @@ rule build_geothermal_heat_potential: heat_source_power=resources( "heat_source_power_geothermal_base_s_{clusters}.csv" ), - resources: - mem_mb=2000, log: logs("build_heat_source_potentials_geothermal_s_{clusters}.log"), benchmark: benchmarks("build_heat_source_potentials/geothermal_s_{clusters}") + resources: + mem_mb=2000, + params: + drop_leap_day=config_provider("enable", "drop_leap_day"), + countries=config_provider("countries"), + constant_temperature_celsius=config_provider( + "sector", + "district_heating", + "limited_heat_sources", + "geothermal", + "constant_temperature_celsius", + ), + ignore_missing_regions=config_provider( + "sector", + "district_heating", + "limited_heat_sources", + "geothermal", + "ignore_missing_regions", + ), + message: + "Building geothermal heat potential estimates for {wildcards.clusters} clusters" script: scripts("build_geothermal_heat_potential.py") rule build_ates_potentials: - message: - "Building aquifer thermal energy storage (ATES) potentials for {wildcards.clusters} clusters and {wildcards.planning_horizons} planning horizon" + input: + aquifer_shapes_shp=rules.retrieve_aquifer_data_bgr.output["aquifer_shapes"][0], + dh_areas=resources("dh_areas_base_s_{clusters}.geojson"), + regions_onshore=resources("regions_onshore_base_s_{clusters}.geojson"), + central_heating_forward_temperature_profiles=resources( + "central_heating_forward_temperature_profiles_base_s_{clusters}_{planning_horizons}.nc" + ), + central_heating_return_temperature_profiles=resources( + "central_heating_return_temperature_profiles_base_s_{clusters}_{planning_horizons}.nc" + ), + output: + ates_potentials=resources( + "ates_potentials_base_s_{clusters}_{planning_horizons}.csv" + ), + log: + logs("build_ates_potentials_s_{clusters}_{planning_horizons}.log"), + benchmark: + benchmarks("build_ates_potentials_geothermal_s_{clusters}_{planning_horizons}") + resources: + mem_mb=2000, params: max_top_temperature=config_provider( "sector", @@ -422,26 +440,8 @@ rule build_ates_potentials: "ignore_missing_regions", ), countries=config_provider("countries"), - input: - aquifer_shapes_shp=rules.retrieve_aquifer_data_bgr.output["aquifer_shapes"][0], - dh_areas=resources("dh_areas_base_s_{clusters}.geojson"), - regions_onshore=resources("regions_onshore_base_s_{clusters}.geojson"), - central_heating_forward_temperature_profiles=resources( - "central_heating_forward_temperature_profiles_base_s_{clusters}_{planning_horizons}.nc" - ), - central_heating_return_temperature_profiles=resources( - "central_heating_return_temperature_profiles_base_s_{clusters}_{planning_horizons}.nc" - ), - output: - ates_potentials=resources( - "ates_potentials_base_s_{clusters}_{planning_horizons}.csv" - ), - resources: - mem_mb=2000, - log: - logs("build_ates_potentials_s_{clusters}_{planning_horizons}.log"), - benchmark: - benchmarks("build_ates_potentials_geothermal_s_{clusters}_{planning_horizons}") + message: + "Building aquifer thermal energy storage (ATES) potentials for {wildcards.clusters} clusters and {wildcards.planning_horizons} planning horizon" script: scripts("build_ates_potentials.py") @@ -489,13 +489,6 @@ def input_hera_data(w) -> dict[str, str]: rule build_river_heat_potential: - params: - drop_leap_day=config_provider("enable", "drop_leap_day"), - snapshots=config_provider("snapshots"), - dh_area_buffer=config_provider( - "sector", "district_heating", "dh_areas", "buffer" - ), - enable_heat_source_maps=config_provider("plotting", "enable_heat_source_maps"), input: unpack(input_hera_data), regions_onshore=resources("regions_onshore_base_s_{clusters}.geojson"), @@ -511,13 +504,20 @@ rule build_river_heat_potential: heat_source_energy_temporal_aggregate=resources( "heat_source_energy_river_water_base_s_{clusters}_temporal_aggregate.nc" ), - resources: - mem_mb=20000, log: logs("build_river_water_heat_potential_base_s_{clusters}.log"), benchmark: benchmarks("build_river_water_heat_potential_base_s_{clusters}") threads: 1 + resources: + mem_mb=20000, + params: + drop_leap_day=config_provider("enable", "drop_leap_day"), + snapshots=config_provider("snapshots"), + dh_area_buffer=config_provider( + "sector", "district_heating", "dh_areas", "buffer" + ), + enable_heat_source_maps=config_provider("plotting", "enable_heat_source_maps"), script: scripts( "build_surface_water_heat_potentials/build_river_water_heat_potential.py" @@ -624,12 +624,6 @@ def input_seawater_temperature(w) -> dict[str, str]: rule build_sea_heat_potential: - params: - drop_leap_day=config_provider("enable", "drop_leap_day"), - snapshots=config_provider("snapshots"), - dh_area_buffer=config_provider( - "sector", "district_heating", "dh_areas", "buffer" - ), input: # seawater_temperature=lambda w: input_seawater_temperature(w), unpack(input_seawater_temperature), @@ -640,35 +634,24 @@ rule build_sea_heat_potential: heat_source_temperature_temporal_aggregate=resources( "temp_sea_water_base_s_{clusters}_temporal_aggregate.nc" ), - resources: - mem_mb=10000, log: logs("build_sea_water_heat_potential_base_s_{clusters}.log"), benchmark: benchmarks("build_sea_water_heat_potential_base_s_{clusters}") threads: config["atlite"].get("nprocesses", 4) + resources: + mem_mb=10000, + params: + drop_leap_day=config_provider("enable", "drop_leap_day"), + snapshots=config_provider("snapshots"), + dh_area_buffer=config_provider( + "sector", "district_heating", "dh_areas", "buffer" + ), script: scripts("build_surface_water_heat_potentials/build_sea_water_heat_potential.py") rule build_cop_profiles: - message: - "Building coefficient of performance (COP) profiles for {wildcards.clusters} clusters and {wildcards.planning_horizons} planning horizon" - params: - heat_pump_sink_T_decentral_heating=config_provider( - "sector", "heat_pump_sink_T_individual_heating" - ), - heat_source_cooling_central_heating=config_provider( - "sector", "district_heating", "heat_source_cooling" - ), - heat_pump_cop_approximation_central_heating=config_provider( - "sector", "district_heating", "heat_pump_cop_approximation" - ), - heat_pump_sources=config_provider("sector", "heat_pump_sources"), - limited_heat_sources=config_provider( - "sector", "district_heating", "limited_heat_sources" - ), - snapshots=config_provider("snapshots"), input: unpack(input_heat_source_temperature), central_heating_forward_temperature_profiles=resources( @@ -684,33 +667,34 @@ rule build_cop_profiles: ), output: cop_profiles=resources("cop_profiles_base_s_{clusters}_{planning_horizons}.nc"), - resources: - mem_mb=20000, log: logs("build_cop_profiles_s_{clusters}_{planning_horizons}.log"), benchmark: benchmarks("build_cop_profiles/s_{clusters}_{planning_horizons}") + resources: + mem_mb=20000, + params: + heat_pump_sink_T_decentral_heating=config_provider( + "sector", "heat_pump_sink_T_individual_heating" + ), + heat_source_cooling_central_heating=config_provider( + "sector", "district_heating", "heat_source_cooling" + ), + heat_pump_cop_approximation_central_heating=config_provider( + "sector", "district_heating", "heat_pump_cop_approximation" + ), + heat_pump_sources=config_provider("sector", "heat_pump_sources"), + limited_heat_sources=config_provider( + "sector", "district_heating", "limited_heat_sources" + ), + snapshots=config_provider("snapshots"), + message: + "Building coefficient of performance (COP) profiles for {wildcards.clusters} clusters and {wildcards.planning_horizons} planning horizon" script: scripts("build_cop_profiles/run.py") rule build_ptes_operations: - message: - "Building thermal energy storage operations profiles for {wildcards.clusters} clusters and {wildcards.planning_horizons} planning horizon" - params: - max_ptes_top_temperature=config_provider( - "sector", - "district_heating", - "ptes", - "max_top_temperature", - ), - min_ptes_bottom_temperature=config_provider( - "sector", - "district_heating", - "ptes", - "min_bottom_temperature", - ), - snapshots=config_provider("snapshots"), input: central_heating_forward_temperature_profiles=resources( "central_heating_forward_temperature_profiles_base_s_{clusters}_{planning_horizons}.nc" @@ -729,27 +713,33 @@ rule build_ptes_operations: ptes_e_max_pu_profiles=resources( "ptes_e_max_pu_profiles_base_s_{clusters}_{planning_horizons}.nc" ), - resources: - mem_mb=2000, log: logs("build_ptes_operations_s_{clusters}_{planning_horizons}.log"), benchmark: benchmarks("build_ptes_operations_s_{clusters}_{planning_horizons}") - script: - scripts("build_ptes_operations/run.py") - - -rule build_direct_heat_source_utilisation_profiles: - message: - "Building direct heat source utilization profiles for industrial applications for {wildcards.clusters} clusters and {wildcards.planning_horizons} planning horizon" + resources: + mem_mb=2000, params: - direct_utilisation_heat_sources=config_provider( - "sector", "district_heating", "direct_utilisation_heat_sources" + max_ptes_top_temperature=config_provider( + "sector", + "district_heating", + "ptes", + "max_top_temperature", ), - limited_heat_sources=config_provider( - "sector", "district_heating", "limited_heat_sources" + min_ptes_bottom_temperature=config_provider( + "sector", + "district_heating", + "ptes", + "min_bottom_temperature", ), snapshots=config_provider("snapshots"), + message: + "Building thermal energy storage operations profiles for {wildcards.clusters} clusters and {wildcards.planning_horizons} planning horizon" + script: + scripts("build_ptes_operations/run.py") + + +rule build_direct_heat_source_utilisation_profiles: input: central_heating_forward_temperature_profiles=resources( "central_heating_forward_temperature_profiles_base_s_{clusters}_{planning_horizons}.nc" @@ -758,8 +748,6 @@ rule build_direct_heat_source_utilisation_profiles: direct_heat_source_utilisation_profiles=resources( "direct_heat_source_utilisation_profiles_base_s_{clusters}_{planning_horizons}.nc" ), - resources: - mem_mb=20000, log: logs( "build_direct_heat_source_utilisation_profiles_s_{clusters}_{planning_horizons}.log" @@ -768,76 +756,83 @@ rule build_direct_heat_source_utilisation_profiles: benchmarks( "build_direct_heat_source_utilisation_profiles/s_{clusters}_{planning_horizons}" ) + resources: + mem_mb=20000, + params: + direct_utilisation_heat_sources=config_provider( + "sector", "district_heating", "direct_utilisation_heat_sources" + ), + limited_heat_sources=config_provider( + "sector", "district_heating", "limited_heat_sources" + ), + snapshots=config_provider("snapshots"), + message: + "Building direct heat source utilization profiles for industrial applications for {wildcards.clusters} clusters and {wildcards.planning_horizons} planning horizon" script: scripts("build_direct_heat_source_utilisation_profiles.py") rule build_solar_thermal_profiles: - message: - "Building solar thermal generation profiles for {wildcards.clusters} clusters" - params: - snapshots=config_provider("snapshots"), - drop_leap_day=config_provider("enable", "drop_leap_day"), - solar_thermal=config_provider("solar_thermal"), input: pop_layout=resources("pop_layout_total.nc"), regions_onshore=resources("regions_onshore_base_s_{clusters}.geojson"), cutout=lambda w: input_cutout(w, config_provider("solar_thermal", "cutout")(w)), output: solar_thermal=resources("solar_thermal_total_base_s_{clusters}.nc"), - resources: - mem_mb=20000, - threads: 16 log: logs("build_solar_thermal_profiles_total_s_{clusters}.log"), benchmark: benchmarks("build_solar_thermal_profiles/total_{clusters}") + threads: 16 + resources: + mem_mb=20000, + params: + snapshots=config_provider("snapshots"), + drop_leap_day=config_provider("enable", "drop_leap_day"), + solar_thermal=config_provider("solar_thermal"), + message: + "Building solar thermal generation profiles for {wildcards.clusters} clusters" script: scripts("build_solar_thermal_profiles.py") rule build_eurostat_balances: - message: - "Building Eurostat energy balances" input: tsv_gz=rules.retrieve_eurostat_balances.output["tsv_gz"], output: csv=resources("eurostat_energy_balances.csv"), - threads: 1 - resources: - mem_mb=4000, log: logs("build_eurostat_balances.log"), benchmark: benchmarks("build_eurostat_balances") + threads: 1 + resources: + mem_mb=4000, + message: + "Building Eurostat energy balances" script: "../scripts/build_eurostat_balances.py" rule build_swiss_energy_balances: - message: - "Building BFE Swiss energy balances" input: xlsx=rules.retrieve_swiss_energy_balances.output["xlsx"], output: csv=resources("switzerland_energy_balances.csv"), - threads: 1 - resources: - mem_mb=4000, log: logs("build_swiss_energy_balances.log"), benchmark: benchmarks("build_swiss_energy_balances") + threads: 1 + resources: + mem_mb=4000, + message: + "Building BFE Swiss energy balances" script: scripts("build_swiss_energy_balances.py") rule build_energy_totals: - message: - "Building energy totals" - params: - countries=config_provider("countries"), - energy=config_provider("energy"), input: nuts3_shapes=resources("nuts3_shapes.geojson"), co2=rules.retrieve_ghg_emissions.output["csv"], @@ -854,13 +849,18 @@ rule build_energy_totals: transport_name=resources("transport_data.csv"), district_heat_share=resources("district_heat_share.csv"), heating_efficiencies=resources("heating_efficiencies.csv"), - threads: 16 - resources: - mem_mb=10000, log: logs("build_energy_totals.log"), benchmark: benchmarks("build_energy_totals") + threads: 16 + resources: + mem_mb=10000, + params: + countries=config_provider("countries"), + energy=config_provider("energy"), + message: + "Building energy totals" script: scripts("build_energy_totals.py") @@ -886,29 +886,25 @@ if (COUNTRY_HDD_DATASET := dataset_version("country_hdd"))["source"] in ["build" rule build_heat_totals: - message: - "Building heat totals" input: hdd=f"{COUNTRY_HDD_DATASET['folder']}/era5-HDD-per-country.csv", energy_totals=resources("energy_totals.csv"), output: heat_totals=resources("heat_totals.csv"), - threads: 1 - resources: - mem_mb=2000, log: logs("build_heat_totals.log"), benchmark: benchmarks("build_heat_totals") + threads: 1 + resources: + mem_mb=2000, + message: + "Building heat totals" script: scripts("build_heat_totals.py") rule build_biomass_potentials: - message: - "Building biomass potential estimates for {wildcards.clusters} clusters and {wildcards.planning_horizons} planning horizon" - params: - biomass=config_provider("biomass"), input: enspreso_biomass=rules.retrieve_enspreso_biomass.output["xlsx"], eurostat=resources("eurostat_energy_balances.csv"), @@ -925,39 +921,41 @@ rule build_biomass_potentials: biomass_potentials=resources( "biomass_potentials_s_{clusters}_{planning_horizons}.csv" ), - threads: 8 - resources: - mem_mb=2000, log: logs("build_biomass_potentials_s_{clusters}_{planning_horizons}.log"), benchmark: benchmarks("build_biomass_potentials_s_{clusters}_{planning_horizons}") + threads: 8 + resources: + mem_mb=2000, + params: + biomass=config_provider("biomass"), + message: + "Building biomass potential estimates for {wildcards.clusters} clusters and {wildcards.planning_horizons} planning horizon" script: scripts("build_biomass_potentials.py") rule build_biomass_transport_costs: - message: - "Building biomass transport cost" input: sc1="data/biomass_transport_costs_supplychain1.csv", sc2="data/biomass_transport_costs_supplychain2.csv", output: biomass_transport_costs=resources("biomass_transport_costs.csv"), - threads: 1 - resources: - mem_mb=1000, log: logs("build_biomass_transport_costs.log"), benchmark: benchmarks("build_biomass_transport_costs") + threads: 1 + resources: + mem_mb=1000, + message: + "Building biomass transport cost" script: scripts("build_biomass_transport_costs.py") rule build_co2_sequestration_potentials: - message: - "Building CO2 sequestration potentials" input: storage_table=rules.retrieve_co2stop.output["storage_table"], storage_map=rules.retrieve_co2stop.output["storage_map"], @@ -967,24 +965,20 @@ rule build_co2_sequestration_potentials: traps_map=rules.retrieve_co2stop.output["traps_map"], output: resources("co2_sequestration_potentials.geojson"), - threads: 1 - resources: - mem_mb=4000, log: logs("build_co2_sequestration_potentials.log"), benchmark: benchmarks("build_co2_sequestration_potentials") + threads: 1 + resources: + mem_mb=4000, + message: + "Building CO2 sequestration potentials" script: scripts("build_co2_sequestration_potentials.py") rule build_clustered_co2_sequestration_potentials: - message: - "Clustering CO2 sequestration potentials for {wildcards.clusters} clusters" - params: - sequestration_potential=config_provider( - "sector", "regional_co2_sequestration_potential" - ), input: sequestration_potential=resources("co2_sequestration_potentials.geojson"), regions_onshore=resources("regions_onshore_base_s_{clusters}.geojson"), @@ -993,82 +987,84 @@ rule build_clustered_co2_sequestration_potentials: sequestration_potential=resources( "co2_sequestration_potential_base_s_{clusters}.csv" ), - threads: 1 - resources: - mem_mb=4000, log: logs("build_clustered_co2_sequestration_potentials_{clusters}.log"), benchmark: benchmarks("build_clustered_co2_sequestration_potentials_{clusters}") + threads: 1 + resources: + mem_mb=4000, + params: + sequestration_potential=config_provider( + "sector", "regional_co2_sequestration_potential" + ), + message: + "Clustering CO2 sequestration potentials for {wildcards.clusters} clusters" script: scripts("build_clustered_co2_sequestration_potentials.py") rule build_salt_cavern_potentials: - message: - "Building salt cavern potential for hydrogen storage for {wildcards.clusters} clusters" input: salt_caverns=rules.retrieve_h2_salt_caverns.output["geojson"], regions_onshore=resources("regions_onshore_base_s_{clusters}.geojson"), regions_offshore=resources("regions_offshore_base_s_{clusters}.geojson"), output: h2_cavern_potential=resources("salt_cavern_potentials_s_{clusters}.csv"), - threads: 1 - resources: - mem_mb=2000, log: logs("build_salt_cavern_potentials_s_{clusters}.log"), benchmark: benchmarks("build_salt_cavern_potentials_s_{clusters}") + threads: 1 + resources: + mem_mb=2000, + message: + "Building salt cavern potential for hydrogen storage for {wildcards.clusters} clusters" script: scripts("build_salt_cavern_potentials.py") rule build_ammonia_production: - message: - "Building ammonia production capacity and location data" input: usgs=rules.retrieve_nitrogen_statistics.output["xlsx"], output: ammonia_production=resources("ammonia_production.csv"), - threads: 1 - resources: - mem_mb=1000, log: logs("build_ammonia_production.log"), benchmark: benchmarks("build_ammonia_production") + threads: 1 + resources: + mem_mb=1000, + message: + "Building ammonia production capacity and location data" script: scripts("build_ammonia_production.py") rule build_industry_sector_ratios: - message: - "Building industry sector energy demand ratios" - params: - industry=config_provider("industry"), - ammonia=config_provider("sector", "ammonia", default=False), input: ammonia_production=resources("ammonia_production.csv"), idees=rules.retrieve_jrc_idees.output["directory"], output: industry_sector_ratios=resources("industry_sector_ratios.csv"), - threads: 1 - resources: - mem_mb=1000, log: logs("build_industry_sector_ratios.log"), benchmark: benchmarks("build_industry_sector_ratios") + threads: 1 + resources: + mem_mb=1000, + params: + industry=config_provider("industry"), + ammonia=config_provider("sector", "ammonia", default=False), + message: + "Building industry sector energy demand ratios" script: scripts("build_industry_sector_ratios.py") rule build_industry_sector_ratios_intermediate: - message: - "Building intermediate industry sector ratios for {wildcards.planning_horizons} planning horizon" - params: - industry=config_provider("industry"), input: industry_sector_ratios=resources("industry_sector_ratios.csv"), industrial_energy_demand_per_country_today=resources( @@ -1081,23 +1077,22 @@ rule build_industry_sector_ratios_intermediate: industry_sector_ratios=resources( "industry_sector_ratios_{planning_horizons}.csv" ), - threads: 1 - resources: - mem_mb=1000, log: logs("build_industry_sector_ratios_{planning_horizons}.log"), benchmark: benchmarks("build_industry_sector_ratios_{planning_horizons}") + threads: 1 + resources: + mem_mb=1000, + params: + industry=config_provider("industry"), + message: + "Building intermediate industry sector ratios for {wildcards.planning_horizons} planning horizon" script: scripts("build_industry_sector_ratios_intermediate.py") rule build_industrial_production_per_country: - message: - "Building industrial production statistics per country" - params: - industry=config_provider("industry"), - countries=config_provider("countries"), input: ch_industrial_production="data/ch_industrial_production_per_subsector.csv", ammonia_production=resources("ammonia_production.csv"), @@ -1107,22 +1102,23 @@ rule build_industrial_production_per_country: industrial_production_per_country=resources( "industrial_production_per_country.csv" ), - threads: 8 - resources: - mem_mb=2000, log: logs("build_industrial_production_per_country.log"), benchmark: benchmarks("build_industrial_production_per_country") + threads: 8 + resources: + mem_mb=2000, + params: + industry=config_provider("industry"), + countries=config_provider("countries"), + message: + "Building industrial production statistics per country" script: scripts("build_industrial_production_per_country.py") rule build_industrial_production_per_country_tomorrow: - message: - "Building future industrial production projections for {wildcards.planning_horizons} planning horizon" - params: - industry=config_provider("industry"), input: industrial_production_per_country=resources( "industrial_production_per_country.csv" @@ -1131,9 +1127,6 @@ rule build_industrial_production_per_country_tomorrow: industrial_production_per_country_tomorrow=resources( "industrial_production_per_country_tomorrow_{planning_horizons}.csv" ), - threads: 1 - resources: - mem_mb=1000, log: logs("build_industrial_production_per_country_tomorrow_{planning_horizons}.log"), benchmark: @@ -1142,18 +1135,18 @@ rule build_industrial_production_per_country_tomorrow: "build_industrial_production_per_country_tomorrow_{planning_horizons}" ) ) + threads: 1 + resources: + mem_mb=1000, + params: + industry=config_provider("industry"), + message: + "Building future industrial production projections for {wildcards.planning_horizons} planning horizon" script: scripts("build_industrial_production_per_country_tomorrow.py") rule build_industrial_distribution_key: - message: - "Building industrial activity distribution mapping key for {wildcards.clusters} clusters" - params: - hotmaps_locate_missing=config_provider( - "industry", "hotmaps_locate_missing", default=False - ), - countries=config_provider("countries"), input: regions_onshore=resources("regions_onshore_base_s_{clusters}.geojson"), clustered_pop_layout=resources("pop_layout_base_s_{clusters}.csv"), @@ -1166,20 +1159,25 @@ rule build_industrial_distribution_key: industrial_distribution_key=resources( "industrial_distribution_key_base_s_{clusters}.csv" ), - threads: 1 - resources: - mem_mb=1000, log: logs("build_industrial_distribution_key_{clusters}.log"), benchmark: benchmarks("build_industrial_distribution_key/s_{clusters}") + threads: 1 + resources: + mem_mb=1000, + params: + hotmaps_locate_missing=config_provider( + "industry", "hotmaps_locate_missing", default=False + ), + countries=config_provider("countries"), + message: + "Building industrial activity distribution mapping key for {wildcards.clusters} clusters" script: scripts("build_industrial_distribution_key.py") rule build_industrial_production_per_node: - message: - "Distributing industrial production to network nodes for {wildcards.clusters} clusters and {wildcards.planning_horizons} planning horizon" input: industrial_distribution_key=resources( "industrial_distribution_key_base_s_{clusters}.csv" @@ -1191,9 +1189,6 @@ rule build_industrial_production_per_node: industrial_production_per_node=resources( "industrial_production_base_s_{clusters}_{planning_horizons}.csv" ), - threads: 1 - resources: - mem_mb=1000, log: logs("build_industrial_production_per_node_{clusters}_{planning_horizons}.log"), benchmark: @@ -1202,13 +1197,16 @@ rule build_industrial_production_per_node: "build_industrial_production_per_node/s_{clusters}_{planning_horizons}" ) ) + threads: 1 + resources: + mem_mb=1000, + message: + "Distributing industrial production to network nodes for {wildcards.clusters} clusters and {wildcards.planning_horizons} planning horizon" script: scripts("build_industrial_production_per_node.py") rule build_industrial_energy_demand_per_node: - message: - "Building industrial energy demand per network node for {wildcards.clusters} clusters and {wildcards.planning_horizons} planning horizon" input: industry_sector_ratios=resources( "industry_sector_ratios_{planning_horizons}.csv" @@ -1223,9 +1221,6 @@ rule build_industrial_energy_demand_per_node: industrial_energy_demand_per_node=resources( "industrial_energy_demand_base_s_{clusters}_{planning_horizons}.csv" ), - threads: 1 - resources: - mem_mb=1000, log: logs( "build_industrial_energy_demand_per_node_{clusters}_{planning_horizons}.log" @@ -1236,17 +1231,16 @@ rule build_industrial_energy_demand_per_node: "build_industrial_energy_demand_per_node/s_{clusters}_{planning_horizons}" ) ) + threads: 1 + resources: + mem_mb=1000, + message: + "Building industrial energy demand per network node for {wildcards.clusters} clusters and {wildcards.planning_horizons} planning horizon" script: scripts("build_industrial_energy_demand_per_node.py") rule build_industrial_energy_demand_per_country_today: - message: - "Building current industrial energy demand by country" - params: - countries=config_provider("countries"), - industry=config_provider("industry"), - ammonia=config_provider("sector", "ammonia", default=False), input: transformation_output_coke=resources("transformation_output_coke.csv"), jrc=rules.retrieve_jrc_idees.output["directory"], @@ -1257,20 +1251,24 @@ rule build_industrial_energy_demand_per_country_today: industrial_energy_demand_per_country_today=resources( "industrial_energy_demand_per_country_today.csv" ), - threads: 8 - resources: - mem_mb=2000, log: logs("build_industrial_energy_demand_per_country_today.log"), benchmark: benchmarks("build_industrial_energy_demand_per_country_today") + threads: 8 + resources: + mem_mb=2000, + params: + countries=config_provider("countries"), + industry=config_provider("industry"), + ammonia=config_provider("sector", "ammonia", default=False), + message: + "Building current industrial energy demand by country" script: scripts("build_industrial_energy_demand_per_country_today.py") rule build_industrial_energy_demand_per_node_today: - message: - "Building current industrial energy demand per network node for {wildcards.clusters} clusters" input: industrial_distribution_key=resources( "industrial_distribution_key_base_s_{clusters}.csv" @@ -1282,23 +1280,20 @@ rule build_industrial_energy_demand_per_node_today: industrial_energy_demand_per_node_today=resources( "industrial_energy_demand_today_base_s_{clusters}.csv" ), - threads: 1 - resources: - mem_mb=1000, log: logs("build_industrial_energy_demand_per_node_today_{clusters}.log"), benchmark: benchmarks("build_industrial_energy_demand_per_node_today/s_{clusters}") + threads: 1 + resources: + mem_mb=1000, + message: + "Building current industrial energy demand per network node for {wildcards.clusters} clusters" script: scripts("build_industrial_energy_demand_per_node_today.py") rule build_retro_cost: - message: - "Building retrofitting cost estimates for building efficiency improvements for {wildcards.clusters} clusters" - params: - retrofitting=config_provider("sector", "retrofitting"), - countries=config_provider("countries"), input: building_stock="data/retro/data_building_stock.csv", data_tabula="data/bundle/retro/tabula-calculator-calcsetbuilding.csv", @@ -1313,57 +1308,62 @@ rule build_retro_cost: output: retro_cost=resources("retro_cost_base_s_{clusters}.csv"), floor_area=resources("floor_area_base_s_{clusters}.csv"), - resources: - mem_mb=1000, log: logs("build_retro_cost_{clusters}.log"), benchmark: benchmarks("build_retro_cost/s_{clusters}") + resources: + mem_mb=1000, + params: + retrofitting=config_provider("sector", "retrofitting"), + countries=config_provider("countries"), + message: + "Building retrofitting cost estimates for building efficiency improvements for {wildcards.clusters} clusters" script: scripts("build_retro_cost.py") rule build_population_weighted_energy_totals: - message: - "Building population-weighted energy demand totals for {wildcards.clusters} clusters" - params: - snapshots=config_provider("snapshots"), - drop_leap_day=config_provider("enable", "drop_leap_day"), input: energy_totals=resources("{kind}_totals.csv"), clustered_pop_layout=resources("pop_layout_base_s_{clusters}.csv"), output: resources("pop_weighted_{kind}_totals_s_{clusters}.csv"), - threads: 1 - resources: - mem_mb=2000, log: logs("build_population_weighted_{kind}_totals_{clusters}.log"), benchmark: benchmarks("build_population_weighted_{kind}_totals_{clusters}") + threads: 1 + resources: + mem_mb=2000, + params: + snapshots=config_provider("snapshots"), + drop_leap_day=config_provider("enable", "drop_leap_day"), + message: + "Building population-weighted energy demand totals for {wildcards.clusters} clusters" script: scripts("build_population_weighted_energy_totals.py") rule build_shipping_demand: - message: - "Building shipping fuel demand projections for {wildcards.clusters} clusters" input: ports=rules.retrieve_attributed_ports.output["json"], scope=resources("europe_shape.geojson"), regions=resources("regions_onshore_base_s_{clusters}.geojson"), demand=resources("energy_totals.csv"), - params: - energy_totals_year=config_provider("energy", "energy_totals_year"), output: resources("shipping_demand_s_{clusters}.csv"), - threads: 1 - resources: - mem_mb=2000, log: logs("build_shipping_demand_s_{clusters}.log"), benchmark: benchmarks("build_shipping_demand/s_{clusters}") + threads: 1 + resources: + mem_mb=2000, + params: + energy_totals_year=config_provider("energy", "energy_totals_year"), + message: + "Building shipping fuel demand projections for {wildcards.clusters} clusters" script: scripts("build_shipping_demand.py") @@ -1371,8 +1371,6 @@ rule build_shipping_demand: if MOBILITY_PROFILES_DATASET["source"] in ["build"]: rule build_mobility_profiles: - params: - sector=config_provider("sector"), input: zip_files=storage( expand( @@ -1385,25 +1383,20 @@ if MOBILITY_PROFILES_DATASET["source"] in ["build"]: raw_files=directory(MOBILITY_PROFILES_DATASET["folder"] / "raw"), kfz=MOBILITY_PROFILES_DATASET["folder"] / "kfz.csv", pkw=MOBILITY_PROFILES_DATASET["folder"] / "pkw.csv", - threads: 1 - resources: - mem_mb=5000, log: logs("build_mobility_profiles.log"), benchmark: benchmarks("build_mobility_profiles") + threads: 1 + resources: + mem_mb=5000, + params: + sector=config_provider("sector"), script: scripts("build_mobility_profiles.py") rule build_transport_demand: - message: - "Building transport energy demand profiles for {wildcards.clusters} clusters" - params: - snapshots=config_provider("snapshots"), - drop_leap_day=config_provider("enable", "drop_leap_day"), - sector=config_provider("sector"), - energy_totals_year=config_provider("energy", "energy_totals_year"), input: network=resources("networks/base_s.nc"), clustered_pop_layout=resources("pop_layout_base_s_{clusters}.csv"), @@ -1419,23 +1412,25 @@ rule build_transport_demand: transport_data=resources("transport_data_s_{clusters}.csv"), avail_profile=resources("avail_profile_s_{clusters}.csv"), dsm_profile=resources("dsm_profile_s_{clusters}.csv"), - threads: 1 - resources: - mem_mb=2000, log: logs("build_transport_demand_s_{clusters}.log"), benchmark: benchmarks("build_transport_demand/s_{clusters}") + threads: 1 + resources: + mem_mb=2000, + params: + snapshots=config_provider("snapshots"), + drop_leap_day=config_provider("enable", "drop_leap_day"), + sector=config_provider("sector"), + energy_totals_year=config_provider("energy", "energy_totals_year"), + message: + "Building transport energy demand profiles for {wildcards.clusters} clusters" script: scripts("build_transport_demand.py") rule build_district_heat_share: - message: - "Building district heating penetration share data for {wildcards.clusters} clusters and {wildcards.planning_horizons} planning horizon" - params: - sector=config_provider("sector"), - energy_totals_year=config_provider("energy", "energy_totals_year"), input: district_heat_share=resources("district_heat_share.csv"), clustered_pop_layout=resources("pop_layout_base_s_{clusters}.csv"), @@ -1443,24 +1438,23 @@ rule build_district_heat_share: district_heat_share=resources( "district_heat_share_base_s_{clusters}_{planning_horizons}.csv" ), - threads: 1 - resources: - mem_mb=1000, log: logs("build_district_heat_share_{clusters}_{planning_horizons}.log"), benchmark: benchmarks("build_district_heat_share_{clusters}_{planning_horizons}") + threads: 1 + resources: + mem_mb=1000, + params: + sector=config_provider("sector"), + energy_totals_year=config_provider("energy", "energy_totals_year"), + message: + "Building district heating penetration share data for {wildcards.clusters} clusters and {wildcards.planning_horizons} planning horizon" script: scripts("build_district_heat_share.py") rule build_existing_heating_distribution: - message: - "Building existing heating technology distribution data for {wildcards.clusters} clusters and {wildcards.planning_horizons} planning horizon" - params: - baseyear=config_provider("scenario", "planning_horizons", 0), - sector=config_provider("sector"), - existing_capacities=config_provider("existing_capacities"), input: existing_heating=resources("existing_heating.csv"), clustered_pop_layout=resources("pop_layout_base_s_{clusters}.csv"), @@ -1474,11 +1468,6 @@ rule build_existing_heating_distribution: existing_heating_distribution=resources( "existing_heating_distribution_base_s_{clusters}_{planning_horizons}.csv" ), - wildcard_constraints: - planning_horizons=config["scenario"]["planning_horizons"][0], #only applies to baseyear - threads: 1 - resources: - mem_mb=2000, log: logs( "build_existing_heating_distribution_base_s_{clusters}_{planning_horizons}.log" @@ -1487,17 +1476,22 @@ rule build_existing_heating_distribution: benchmarks( "build_existing_heating_distribution/base_s_{clusters}_{planning_horizons}" ) + wildcard_constraints: + planning_horizons=config["scenario"]["planning_horizons"][0], #only applies to baseyear + threads: 1 + resources: + mem_mb=2000, + params: + baseyear=config_provider("scenario", "planning_horizons", 0), + sector=config_provider("sector"), + existing_capacities=config_provider("existing_capacities"), + message: + "Building existing heating technology distribution data for {wildcards.clusters} clusters and {wildcards.planning_horizons} planning horizon" script: scripts("build_existing_heating_distribution.py") rule time_aggregation: - message: - "Performing time series aggregation for temporal resolution reduction for {wildcards.clusters} clusters and {wildcards.opts} electric options and {wildcards.sector_opts} sector options" - params: - time_resolution=config_provider("clustering", "temporal"), - drop_leap_day=config_provider("enable", "drop_leap_day"), - solver_name=config_provider("solving", "solver", "name"), input: network=resources("networks/base_s_{clusters}_elec_{opts}.nc"), hourly_heat_demand_total=lambda w: ( @@ -1514,13 +1508,19 @@ rule time_aggregation: snapshot_weightings=resources( "snapshot_weightings_base_s_{clusters}_elec_{opts}_{sector_opts}.csv" ), - threads: 1 - resources: - mem_mb=5000, log: logs("time_aggregation_base_s_{clusters}_elec_{opts}_{sector_opts}.log"), benchmark: benchmarks("time_aggregation_base_s_{clusters}_elec_{opts}_{sector_opts}") + threads: 1 + resources: + mem_mb=5000, + params: + time_resolution=config_provider("clustering", "temporal"), + drop_leap_day=config_provider("enable", "drop_leap_day"), + solver_name=config_provider("solving", "solver", "name"), + message: + "Performing time series aggregation for temporal resolution reduction for {wildcards.clusters} clusters and {wildcards.opts} electric options and {wildcards.sector_opts} sector options" script: scripts("time_aggregation.py") @@ -1534,13 +1534,6 @@ def input_profile_offwind(w): rule build_egs_potentials: - message: - "Building enhanced geothermal system (EGS) potential estimates for {wildcards.clusters} clusters" - params: - snapshots=config_provider("snapshots"), - drop_leap_day=config_provider("enable", "drop_leap_day"), - sector=config_provider("sector"), - costs=config_provider("costs"), input: egs_cost="data/egs_costs.json", regions=resources("regions_onshore_base_s_{clusters}.geojson"), @@ -1553,13 +1546,20 @@ rule build_egs_potentials: egs_potentials=resources("egs_potentials_{clusters}.csv"), egs_overlap=resources("egs_overlap_{clusters}.csv"), egs_capacity_factors=resources("egs_capacity_factors_{clusters}.csv"), - threads: 1 - resources: - mem_mb=2000, log: logs("build_egs_potentials_{clusters}.log"), benchmark: benchmarks("build_egs_potentials_{clusters}") + threads: 1 + resources: + mem_mb=2000, + params: + snapshots=config_provider("snapshots"), + drop_leap_day=config_provider("enable", "drop_leap_day"), + sector=config_provider("sector"), + costs=config_provider("costs"), + message: + "Building enhanced geothermal system (EGS) potential estimates for {wildcards.clusters} clusters" script: scripts("build_egs_potentials.py") @@ -1581,41 +1581,6 @@ def input_heat_source_power(w): rule prepare_sector_network: - message: - "Preparing integrated sector-coupled energy network for {wildcards.clusters} clusters, {wildcards.planning_horizons} planning horizon, {wildcards.opts} electric options and {wildcards.sector_opts} sector options" - params: - time_resolution=config_provider("clustering", "temporal", "resolution_sector"), - co2_budget=config_provider("co2_budget"), - conventional_carriers=config_provider( - "existing_capacities", "conventional_carriers" - ), - foresight=config_provider("foresight"), - sector=config_provider("sector"), - industry=config_provider("industry"), - renewable=config_provider("renewable"), - lines=config_provider("lines"), - pypsa_eur=config_provider("pypsa_eur"), - length_factor=config_provider("lines", "length_factor"), - planning_horizons=config_provider("scenario", "planning_horizons"), - countries=config_provider("countries"), - adjustments=config_provider("adjustments", "sector"), - emissions_scope=config_provider("energy", "emissions"), - emission_prices=config_provider("costs", "emission_prices"), - electricity=config_provider("electricity"), - biomass=config_provider("biomass"), - RDIR=RDIR, - heat_pump_sources=config_provider("sector", "heat_pump_sources"), - heat_systems=config_provider("sector", "heat_systems"), - energy_totals_year=config_provider("energy", "energy_totals_year"), - direct_utilisation_heat_sources=config_provider( - "sector", "district_heating", "direct_utilisation_heat_sources" - ), - limited_heat_sources=config_provider( - "sector", "district_heating", "limited_heat_sources" - ), - temperature_limited_stores=config_provider( - "sector", "district_heating", "temperature_limited_stores" - ), input: unpack(input_profile_offwind), unpack(input_heat_source_power), @@ -1746,9 +1711,6 @@ rule prepare_sector_network: resources( "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc" ), - threads: 1 - resources: - mem_mb=2000, log: logs( "prepare_sector_network_base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.log" @@ -1757,5 +1719,43 @@ rule prepare_sector_network: benchmarks( "prepare_sector_network/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}" ) + threads: 1 + resources: + mem_mb=2000, + params: + time_resolution=config_provider("clustering", "temporal", "resolution_sector"), + co2_budget=config_provider("co2_budget"), + conventional_carriers=config_provider( + "existing_capacities", "conventional_carriers" + ), + foresight=config_provider("foresight"), + sector=config_provider("sector"), + industry=config_provider("industry"), + renewable=config_provider("renewable"), + lines=config_provider("lines"), + pypsa_eur=config_provider("pypsa_eur"), + length_factor=config_provider("lines", "length_factor"), + planning_horizons=config_provider("scenario", "planning_horizons"), + countries=config_provider("countries"), + adjustments=config_provider("adjustments", "sector"), + emissions_scope=config_provider("energy", "emissions"), + emission_prices=config_provider("costs", "emission_prices"), + electricity=config_provider("electricity"), + biomass=config_provider("biomass"), + RDIR=RDIR, + heat_pump_sources=config_provider("sector", "heat_pump_sources"), + heat_systems=config_provider("sector", "heat_systems"), + energy_totals_year=config_provider("energy", "energy_totals_year"), + direct_utilisation_heat_sources=config_provider( + "sector", "district_heating", "direct_utilisation_heat_sources" + ), + limited_heat_sources=config_provider( + "sector", "district_heating", "limited_heat_sources" + ), + temperature_limited_stores=config_provider( + "sector", "district_heating", "temperature_limited_stores" + ), + message: + "Preparing integrated sector-coupled energy network for {wildcards.clusters} clusters, {wildcards.planning_horizons} planning horizon, {wildcards.opts} electric options and {wildcards.sector_opts} sector options" script: scripts("prepare_sector_network.py") diff --git a/rules/collect.smk b/rules/collect.smk index dd805620c..f6f14c1c8 100644 --- a/rules/collect.smk +++ b/rules/collect.smk @@ -31,30 +31,28 @@ rule process_costs: rule cluster_networks: - message: - "Collecting clustered network files" input: expand( resources("networks/base_s_{clusters}.nc"), **config["scenario"], run=config["run"]["name"], ), + message: + "Collecting clustered network files" rule prepare_elec_networks: - message: - "Collecting prepared electricity network files" input: expand( resources("networks/base_s_{clusters}_elec_{opts}.nc"), **config["scenario"], run=config["run"]["name"], ), + message: + "Collecting prepared electricity network files" rule prepare_sector_networks: - message: - "Collecting prepared sector-coupled network files" input: expand( resources( @@ -63,22 +61,22 @@ rule prepare_sector_networks: **config["scenario"], run=config["run"]["name"], ), + message: + "Collecting prepared sector-coupled network files" rule solve_elec_networks: - message: - "Collecting solved electricity network files" input: expand( RESULTS + "networks/base_s_{clusters}_elec_{opts}.nc", **config["scenario"], run=config["run"]["name"], ), + message: + "Collecting solved electricity network files" rule solve_sector_networks: - message: - "Collecting solved sector-coupled network files" input: expand( RESULTS @@ -86,11 +84,11 @@ rule solve_sector_networks: **config["scenario"], run=config["run"]["name"], ), + message: + "Collecting solved sector-coupled network files" rule solve_sector_networks_perfect: - message: - "Collecting solved sector-coupled network files with perfect foresight" input: expand( RESULTS @@ -98,6 +96,8 @@ rule solve_sector_networks_perfect: **config["scenario"], run=config["run"]["name"], ), + message: + "Collecting solved sector-coupled network files with perfect foresight" def balance_map_paths(kind, w): @@ -117,11 +117,11 @@ def balance_map_paths(kind, w): rule plot_balance_maps: - message: - "Plotting energy balance maps" input: static=lambda w: balance_map_paths("static", w), interactive=lambda w: balance_map_paths("interactive", w), + message: + "Plotting energy balance maps" rule plot_balance_maps_static: @@ -135,8 +135,6 @@ rule plot_balance_maps_interactive: rule plot_power_networks_clustered: - message: - "Plotting clustered power network topology" input: [ expand( @@ -155,3 +153,5 @@ rule plot_power_networks_clustered: run=config["run"]["name"], ), ], + message: + "Plotting clustered power network topology" diff --git a/rules/development.smk b/rules/development.smk index 5bec30321..82ed4ce5e 100644 --- a/rules/development.smk +++ b/rules/development.smk @@ -4,17 +4,6 @@ rule base_network_incumbent: - message: - "Building base network to which to compare against." - params: - countries=config_provider("countries"), - snapshots=config_provider("snapshots"), - drop_leap_day=config_provider("enable", "drop_leap_day"), - lines=config_provider("lines"), - links=config_provider("links"), - transformers=config_provider("transformers"), - clustering=config_provider("clustering", "mode"), - admin_levels=config_provider("clustering", "administrative"), input: unpack(input_base_network_incumbent), nuts3_shapes=resources("nuts3_shapes.geojson"), @@ -33,20 +22,22 @@ rule base_network_incumbent: threads: 4 resources: mem_mb=2000, + params: + countries=config_provider("countries"), + snapshots=config_provider("snapshots"), + drop_leap_day=config_provider("enable", "drop_leap_day"), + lines=config_provider("lines"), + links=config_provider("links"), + transformers=config_provider("transformers"), + clustering=config_provider("clustering", "mode"), + admin_levels=config_provider("clustering", "administrative"), + message: + "Building base network to which to compare against." script: "../scripts/base_network.py" rule make_network_comparison: - message: - "Create network comparison between two PyPSA networks." - params: - countries=config_provider("countries"), - base_network=config_provider("electricity", "base_network"), - compare_to_version=config_provider( - "osm_network_release", "compare_to", "version" - ), - voltages=config_provider("electricity", "voltages"), input: n_release=resources("networks/base.nc"), n_incumbent=resources("osm/comparison/incumbent/networks/base.nc"), @@ -61,18 +52,20 @@ rule make_network_comparison: threads: 1 resources: mem_mb=2000, + params: + countries=config_provider("countries"), + base_network=config_provider("electricity", "base_network"), + compare_to_version=config_provider( + "osm_network_release", "compare_to", "version" + ), + voltages=config_provider("electricity", "voltages"), + message: + "Create network comparison between two PyPSA networks." script: "../scripts/make_network_comparison.py" rule prepare_osm_network_release: - message: - "Preparing OSM network release files and map." - params: - line_types=config["lines"]["types"], - release_version=config_provider("osm_network_release", "release_version"), - include_polygons=True, - export=True, input: base_network=resources("networks/base.nc"), stations_polygon=resources("osm/build/geojson/stations_polygon.geojson"), @@ -91,18 +84,18 @@ rule prepare_osm_network_release: threads: 1 resources: mem_mb=1000, + params: + line_types=config["lines"]["types"], + release_version=config_provider("osm_network_release", "release_version"), + include_polygons=True, + export=True, + message: + "Preparing OSM network release files and map." script: "../scripts/prepare_osm_network_release.py" rule map_incumbent: - message: - "Preparing map of incumbent network for comparison with OSM release." - params: - line_types=config["lines"]["types"], - release_version="Incumbent", - include_polygons=False, - export=False, input: base_network=resources("osm/comparison/incumbent/networks/base.nc"), output: @@ -114,13 +107,18 @@ rule map_incumbent: threads: 1 resources: mem_mb=1000, + params: + line_types=config["lines"]["types"], + release_version="Incumbent", + include_polygons=False, + export=False, + message: + "Preparing map of incumbent network for comparison with OSM release." script: "../scripts/prepare_osm_network_release.py" rule osm_release: - message: - "Creating OSM network release files, map and comparison with incumbent network." input: resources("osm/release/buses.csv"), resources("osm/release/converters.csv"), @@ -130,3 +128,5 @@ rule osm_release: resources("osm/release/map.html"), resources("osm/comparison/map_incumbent.html"), resources("osm/comparison/lengths.pdf"), + message: + "Creating OSM network release files, map and comparison with incumbent network." diff --git a/rules/postprocess.smk b/rules/postprocess.smk index 09d22344e..1ffb1df50 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -6,47 +6,42 @@ if config["foresight"] != "perfect": rule plot_base_network: - message: - "Plotting base power network" - params: - plotting=config_provider("plotting"), input: network=resources("networks/base.nc"), regions_onshore=resources("regions_onshore.geojson"), output: map=resources("maps/power-network.pdf"), + benchmark: + benchmarks("plot_base_network/base") threads: 1 resources: mem_mb=4000, - benchmark: - benchmarks("plot_base_network/base") + params: + plotting=config_provider("plotting"), + message: + "Plotting base power network" script: scripts("plot_base_network.py") rule plot_power_network_clustered: - message: - "Plotting clustered power network for {wildcards.clusters} clusters" - params: - plotting=config_provider("plotting"), input: network=resources("networks/base_s_{clusters}.nc"), regions_onshore=resources("regions_onshore_base_s_{clusters}.geojson"), output: map=resources("maps/power-network-s-{clusters}.pdf"), + benchmark: + benchmarks("plot_power_network_clustered/base_s_{clusters}") threads: 1 resources: mem_mb=4000, - benchmark: - benchmarks("plot_power_network_clustered/base_s_{clusters}") + params: + plotting=config_provider("plotting"), + message: + "Plotting clustered power network for {wildcards.clusters} clusters" script: scripts("plot_power_network_clustered.py") rule plot_power_network: - message: - "Plotting power network for {wildcards.clusters} clusters, {wildcards.opts} electric options, {wildcards.sector_opts} sector options and {wildcards.planning_horizons} planning horizons" - params: - plotting=config_provider("plotting"), - transmission_limit=config_provider("electricity", "transmission_limit"), input: network=RESULTS + "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc", @@ -54,9 +49,6 @@ if config["foresight"] != "perfect": output: map=RESULTS + "maps/static/base_s_{clusters}_{opts}_{sector_opts}-costs-all_{planning_horizons}.pdf", - threads: 2 - resources: - mem_mb=10000, log: RESULTS + "logs/plot_power_network/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.log", @@ -65,15 +57,18 @@ if config["foresight"] != "perfect": RESULTS + "benchmarks/plot_power_network/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}" ) + threads: 2 + resources: + mem_mb=10000, + params: + plotting=config_provider("plotting"), + transmission_limit=config_provider("electricity", "transmission_limit"), + message: + "Plotting power network for {wildcards.clusters} clusters, {wildcards.opts} electric options, {wildcards.sector_opts} sector options and {wildcards.planning_horizons} planning horizons" script: scripts("plot_power_network.py") rule plot_hydrogen_network: - message: - "Plotting hydrogen network for {wildcards.clusters} clusters, {wildcards.opts} electric options, {wildcards.sector_opts} sector options and {wildcards.planning_horizons} planning horizons" - params: - plotting=config_provider("plotting"), - foresight=config_provider("foresight"), input: network=RESULTS + "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc", @@ -81,9 +76,6 @@ if config["foresight"] != "perfect": output: map=RESULTS + "maps/static/base_s_{clusters}_{opts}_{sector_opts}-h2_network_{planning_horizons}.pdf", - threads: 2 - resources: - mem_mb=10000, log: RESULTS + "logs/plot_hydrogen_network/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.log", @@ -92,14 +84,18 @@ if config["foresight"] != "perfect": RESULTS + "benchmarks/plot_hydrogen_network/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}" ) + threads: 2 + resources: + mem_mb=10000, + params: + plotting=config_provider("plotting"), + foresight=config_provider("foresight"), + message: + "Plotting hydrogen network for {wildcards.clusters} clusters, {wildcards.opts} electric options, {wildcards.sector_opts} sector options and {wildcards.planning_horizons} planning horizons" script: scripts("plot_hydrogen_network.py") rule plot_gas_network: - message: - "Plotting methane network for {wildcards.clusters} clusters, {wildcards.opts} electric options, {wildcards.sector_opts} sector options and {wildcards.planning_horizons} planning horizon" - params: - plotting=config_provider("plotting"), input: network=RESULTS + "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc", @@ -107,9 +103,6 @@ if config["foresight"] != "perfect": output: map=RESULTS + "maps/static/base_s_{clusters}_{opts}_{sector_opts}-ch4_network_{planning_horizons}.pdf", - threads: 2 - resources: - mem_mb=10000, log: RESULTS + "logs/plot_gas_network/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.log", @@ -118,15 +111,17 @@ if config["foresight"] != "perfect": RESULTS + "benchmarks/plot_gas_network/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}" ) + threads: 2 + resources: + mem_mb=10000, + params: + plotting=config_provider("plotting"), + message: + "Plotting methane network for {wildcards.clusters} clusters, {wildcards.opts} electric options, {wildcards.sector_opts} sector options and {wildcards.planning_horizons} planning horizon" script: scripts("plot_gas_network.py") rule plot_balance_map: - message: - "Plotting balance map for {wildcards.clusters} clusters, {wildcards.opts} electric options, {wildcards.sector_opts} sector options, {wildcards.planning_horizons} planning horizons and {wildcards.carrier} carrier" - params: - plotting=config_provider("plotting"), - settings=lambda w: config_provider("plotting", "balance_map", w.carrier), input: network=RESULTS + "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc", @@ -134,9 +129,6 @@ if config["foresight"] != "perfect": output: RESULTS + "maps/static/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}-balance_map_{carrier}.pdf", - threads: 1 - resources: - mem_mb=8000, log: RESULTS + "logs/plot_balance_map/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}_{carrier}.log", @@ -145,14 +137,18 @@ if config["foresight"] != "perfect": RESULTS + "benchmarks/plot_balance_map/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}_{carrier}" ) + threads: 1 + resources: + mem_mb=8000, + params: + plotting=config_provider("plotting"), + settings=lambda w: config_provider("plotting", "balance_map", w.carrier), + message: + "Plotting balance map for {wildcards.clusters} clusters, {wildcards.opts} electric options, {wildcards.sector_opts} sector options, {wildcards.planning_horizons} planning horizons and {wildcards.carrier} carrier" script: scripts("plot_balance_map.py") rule plot_balance_map_interactive: - params: - settings=lambda w: config_provider( - "plotting", "balance_map_interactive", w.carrier - ), input: network=RESULTS + "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc", @@ -160,9 +156,6 @@ if config["foresight"] != "perfect": output: RESULTS + "maps/interactive/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}-balance_map_{carrier}.html", - threads: 1 - resources: - mem_mb=8000, log: RESULTS + "logs/plot_balance_map_interactive/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}_{carrier}.log", @@ -171,13 +164,17 @@ if config["foresight"] != "perfect": RESULTS + "benchmarks/plot_interactive_map/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}_{carrier}" ) + threads: 1 + resources: + mem_mb=8000, + params: + settings=lambda w: config_provider( + "plotting", "balance_map_interactive", w.carrier + ), script: scripts("plot_balance_map_interactive.py") rule plot_heat_source_map: - params: - plotting=config_provider("plotting"), - heat_sources=config_provider("sector", "heat_pump_sources"), input: regions=resources("regions_onshore_base_s_{clusters}.geojson"), heat_source_temperature=lambda w: ( @@ -201,9 +198,6 @@ if config["foresight"] != "perfect": + "maps/static/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}-heat_source_temperature_map_{carrier}.html", energy_map=RESULTS + "maps/static/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}-heat_source_energy_map_{carrier}.html", - threads: 1 - resources: - mem_mb=150000, log: RESULTS + "logs/plot_heat_source_map/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}_{carrier}.log", @@ -212,6 +206,12 @@ if config["foresight"] != "perfect": RESULTS + "benchmarks/plot_heat_source_map/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}_{carrier}" ) + threads: 1 + resources: + mem_mb=150000, + params: + plotting=config_provider("plotting"), + heat_sources=config_provider("sector", "heat_pump_sources"), script: scripts("plot_heat_source_map.py") @@ -227,10 +227,6 @@ if config["foresight"] == "perfect": } rule plot_power_network_perfect: - message: - "Plotting power network with perfect foresight for {wildcards.clusters} clusters, {wildcards.opts} electric options and {wildcards.sector_opts} sector options" - params: - plotting=config_provider("plotting"), input: network=RESULTS + "networks/base_s_{clusters}_{opts}_{sector_opts}_brownfield_all_years.nc", @@ -240,13 +236,15 @@ if config["foresight"] == "perfect": threads: 2 resources: mem_mb=10000, + params: + plotting=config_provider("plotting"), + message: + "Plotting power network with perfect foresight for {wildcards.clusters} clusters, {wildcards.opts} electric options and {wildcards.sector_opts} sector options" script: scripts("plot_power_network_perfect.py") rule make_summary: - message: - "Creating optimization results summary statistics" input: network=RESULTS + "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc", @@ -279,9 +277,6 @@ rule make_summary: + "csvs/individual/market_values_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.csv", metrics=RESULTS + "csvs/individual/metrics_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.csv", - threads: 1 - resources: - mem_mb=8000, log: RESULTS + "logs/make_summary_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.log", @@ -290,16 +285,16 @@ rule make_summary: RESULTS + "benchmarks/make_summary_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}" ) + threads: 1 + resources: + mem_mb=8000, + message: + "Creating optimization results summary statistics" script: scripts("make_summary.py") rule make_global_summary: - message: - "Creating global summary of optimization results for all scenarios" - params: - scenario=config_provider("scenario"), - RDIR=RDIR, input: nodal_costs=expand( RESULTS @@ -400,49 +395,43 @@ rule make_global_summary: nodal_capacities=RESULTS + "csvs/nodal_capacities.csv", nodal_energy_balance=RESULTS + "csvs/nodal_energy_balance.csv", nodal_capacity_factors=RESULTS + "csvs/nodal_capacity_factors.csv", - threads: 1 - resources: - mem_mb=8000, log: RESULTS + "logs/make_global_summary.log", benchmark: RESULTS + "benchmarks/make_global_summary" + threads: 1 + resources: + mem_mb=8000, + params: + scenario=config_provider("scenario"), + RDIR=RDIR, + message: + "Creating global summary of optimization results for all scenarios" script: scripts("make_global_summary.py") rule make_cumulative_costs: - message: - "Calculating cumulative costs over time horizon" - params: - scenario=config_provider("scenario"), input: costs=RESULTS + "csvs/costs.csv", output: cumulative_costs=RESULTS + "csvs/cumulative_costs.csv", - threads: 1 - resources: - mem_mb=4000, log: RESULTS + "logs/make_cumulative_costs.log", benchmark: RESULTS + "benchmarks/make_cumulative_costs" + threads: 1 + resources: + mem_mb=4000, + params: + scenario=config_provider("scenario"), + message: + "Calculating cumulative costs over time horizon" script: scripts("make_cumulative_costs.py") rule plot_summary: - message: - "Plotting summary statistics and results" - params: - countries=config_provider("countries"), - planning_horizons=config_provider("scenario", "planning_horizons"), - emissions_scope=config_provider("energy", "emissions"), - plotting=config_provider("plotting"), - foresight=config_provider("foresight"), - co2_budget=config_provider("co2_budget"), - sector=config_provider("sector"), - RDIR=RDIR, input: costs=RESULTS + "csvs/costs.csv", energy=RESULTS + "csvs/energy.csv", @@ -453,69 +442,80 @@ rule plot_summary: costs=RESULTS + "graphs/costs.pdf", energy=RESULTS + "graphs/energy.pdf", balances=RESULTS + "graphs/balances-energy.pdf", + log: + RESULTS + "logs/plot_summary.log", threads: 2 resources: mem_mb=10000, - log: - RESULTS + "logs/plot_summary.log", + params: + countries=config_provider("countries"), + planning_horizons=config_provider("scenario", "planning_horizons"), + emissions_scope=config_provider("energy", "emissions"), + plotting=config_provider("plotting"), + foresight=config_provider("foresight"), + co2_budget=config_provider("co2_budget"), + sector=config_provider("sector"), + RDIR=RDIR, + message: + "Plotting summary statistics and results" script: scripts("plot_summary.py") rule plot_balance_timeseries: - message: - "Plotting energy balance time series for {wildcards.clusters} clusters, {wildcards.opts} electric options, {wildcards.sector_opts} sector options and {wildcards.planning_horizons} planning horizons" - params: - plotting=config_provider("plotting"), - snapshots=config_provider("snapshots"), - drop_leap_day=config_provider("enable", "drop_leap_day"), input: network=RESULTS + "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc", rc="matplotlibrc", - threads: 16 - resources: - mem_mb=10000, - log: - RESULTS - + "logs/plot_balance_timeseries/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.log", - benchmark: - RESULTS - +"benchmarks/plot_balance_timeseries/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}" output: directory( RESULTS + "graphics/balance_timeseries/s_{clusters}_{opts}_{sector_opts}_{planning_horizons}" ), + log: + RESULTS + + "logs/plot_balance_timeseries/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.log", + benchmark: + RESULTS + + "benchmarks/plot_balance_timeseries/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}" + threads: 16 + resources: + mem_mb=10000, + params: + plotting=config_provider("plotting"), + snapshots=config_provider("snapshots"), + drop_leap_day=config_provider("enable", "drop_leap_day"), + message: + "Plotting energy balance time series for {wildcards.clusters} clusters, {wildcards.opts} electric options, {wildcards.sector_opts} sector options and {wildcards.planning_horizons} planning horizons" script: scripts("plot_balance_timeseries.py") rule plot_heatmap_timeseries: - message: - "Plotting heatmap time series visualization for {wildcards.clusters} clusters, {wildcards.opts} electric options, {wildcards.sector_opts} sector options and {wildcards.planning_horizons} planning horizons" - params: - plotting=config_provider("plotting"), - snapshots=config_provider("snapshots"), - drop_leap_day=config_provider("enable", "drop_leap_day"), input: network=RESULTS + "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc", rc="matplotlibrc", - threads: 16 - resources: - mem_mb=10000, - log: - RESULTS - + "logs/plot_heatmap_timeseries/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.log", - benchmark: - RESULTS - +"benchmarks/plot_heatmap_timeseries/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}" output: directory( RESULTS + "graphics/heatmap_timeseries/s_{clusters}_{opts}_{sector_opts}_{planning_horizons}" ), + log: + RESULTS + + "logs/plot_heatmap_timeseries/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.log", + benchmark: + RESULTS + + "benchmarks/plot_heatmap_timeseries/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}" + threads: 16 + resources: + mem_mb=10000, + params: + plotting=config_provider("plotting"), + snapshots=config_provider("snapshots"), + drop_leap_day=config_provider("enable", "drop_leap_day"), + message: + "Plotting heatmap time series visualization for {wildcards.clusters} clusters, {wildcards.opts} electric options, {wildcards.sector_opts} sector options and {wildcards.planning_horizons} planning horizons" script: scripts("plot_heatmap_timeseries.py") @@ -548,8 +548,6 @@ STATISTICS = { rule write_statistics: - params: - statistics=STATISTICS, input: network=RESULTS + "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc", @@ -566,14 +564,13 @@ rule write_statistics: log: RESULTS + "logs/write_statistics/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}_country-{country}_carrier-{carrier}.log", + params: + statistics=STATISTICS, script: "../scripts/write_statistics.py" rule plot_statistics_single: - params: - plotting=config["plotting"], - statistics=STATISTICS, input: **{ f"{csv}": RESULTS @@ -595,14 +592,14 @@ rule plot_statistics_single: log: RESULTS + "logs/plot_statistics_single/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}_country-{country}_carrier-{carrier}.log", + params: + plotting=config["plotting"], + statistics=STATISTICS, script: "../scripts/plot_statistics_single.py" rule plot_statistics_comparison: - params: - plotting=config["plotting"], - statistics=STATISTICS, input: expand( RESULTS @@ -624,16 +621,14 @@ rule plot_statistics_comparison: log: RESULTS + "logs/plot_statistics_comparison/country-{country}_carrier-{carrier}.log", + params: + plotting=config["plotting"], + statistics=STATISTICS, script: "../scripts/plot_statistics_comparison.py" rule plot_base_statistics: - message: - "Plotting base scenario statistics for {wildcards.clusters} clusters and {wildcards.opts} electric options" - params: - plotting=config_provider("plotting"), - barplots=STATISTICS_BARPLOTS, input: network=RESULTS + "networks/base_s_{clusters}_elec_{opts}.nc", output: @@ -644,6 +639,11 @@ rule plot_base_statistics: }, barplots_touch=RESULTS + "figures/.statistics_plots_base_s_{clusters}_elec_{opts}", + params: + plotting=config_provider("plotting"), + barplots=STATISTICS_BARPLOTS, + message: + "Plotting base scenario statistics for {wildcards.clusters} clusters and {wildcards.opts} electric options" script: scripts("plot_statistics.py") @@ -656,9 +656,6 @@ rule build_ambient_air_temperature_yearly_average: average_ambient_air_temperature=resources( "temp_ambient_air_base_s_{clusters}_temporal_aggregate.nc" ), - threads: 1 - resources: - mem_mb=5000, log: RESULTS + "logs/build_ambient_air_temperature_yearly_average/base_s_{clusters}", benchmark: @@ -666,6 +663,9 @@ rule build_ambient_air_temperature_yearly_average: RESULTS + "benchmarks/build_ambient_air_temperature_yearly_average/base_s_{clusters}" ) + threads: 1 + resources: + mem_mb=5000, script: scripts("build_ambient_air_temperature_yearly_average.py") @@ -686,13 +686,6 @@ rule plot_cop_profiles: rule plot_interactive_bus_balance: - params: - plotting=config_provider("plotting"), - snapshots=config_provider("snapshots"), - drop_leap_day=config_provider("enable", "drop_leap_day"), - bus_name_pattern=config_provider( - "plotting", "interactive_bus_balance", "bus_name_pattern" - ), input: network=RESULTS + "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc", @@ -707,8 +700,15 @@ rule plot_interactive_bus_balance: + "logs/plot_interactive_bus_balance/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.log", benchmark: RESULTS - +"benchmarks/plot_interactive_bus_balance/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}" + + "benchmarks/plot_interactive_bus_balance/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}" resources: mem_mb=20000, + params: + plotting=config_provider("plotting"), + snapshots=config_provider("snapshots"), + drop_leap_day=config_provider("enable", "drop_leap_day"), + bus_name_pattern=config_provider( + "plotting", "interactive_bus_balance", "bus_name_pattern" + ), script: scripts("plot_interactive_bus_balance.py") diff --git a/rules/pypsa-de/modifications.smk b/rules/pypsa-de/modifications.smk index a9f8fdb5d..93c86bf2e 100644 --- a/rules/pypsa-de/modifications.smk +++ b/rules/pypsa-de/modifications.smk @@ -4,9 +4,6 @@ rule build_scenarios: - params: - scenarios=config["run"]["name"], - leitmodelle=config["pypsa-de"]["leitmodelle"], input: ariadne_database="data/ariadne_database.csv", scenario_yaml=config["run"]["scenarios"]["manual_file"], @@ -14,20 +11,14 @@ rule build_scenarios: scenario_yaml=config["run"]["scenarios"]["file"], log: "logs/build_scenarios.log", + params: + scenarios=config["run"]["name"], + leitmodelle=config["pypsa-de"]["leitmodelle"], script: scripts("pypsa-de/build_scenarios.py") rule build_exogenous_mobility_data: - params: - reference_scenario=config_provider("pypsa-de", "reference_scenario"), - planning_horizons=config_provider("scenario", "planning_horizons"), - leitmodelle=config_provider("pypsa-de", "leitmodelle"), - ageb_for_mobility=config_provider("pypsa-de", "ageb_for_mobility"), - uba_for_mobility=config_provider("pypsa-de", "uba_for_mobility"), - shipping_oil_share=config_provider("sector", "shipping_oil_share"), - aviation_demand_factor=config_provider("sector", "aviation_demand_factor"), - energy_totals_year=config_provider("energy", "energy_totals_year"), input: ariadne="data/ariadne_database.csv", energy_totals=resources("energy_totals.csv"), @@ -35,10 +26,19 @@ rule build_exogenous_mobility_data: mobility_data=resources( "modified_mobility_data_{clusters}_{planning_horizons}.csv" ), - resources: - mem_mb=1000, log: logs("build_exogenous_mobility_data_{clusters}_{planning_horizons}.log"), + resources: + mem_mb=1000, + params: + reference_scenario=config_provider("pypsa-de", "reference_scenario"), + planning_horizons=config_provider("scenario", "planning_horizons"), + leitmodelle=config_provider("pypsa-de", "leitmodelle"), + ageb_for_mobility=config_provider("pypsa-de", "ageb_for_mobility"), + uba_for_mobility=config_provider("pypsa-de", "uba_for_mobility"), + shipping_oil_share=config_provider("sector", "shipping_oil_share"), + aviation_demand_factor=config_provider("sector", "aviation_demand_factor"), + energy_totals_year=config_provider("energy", "energy_totals_year"), script: scripts("pypsa-de/build_exogenous_mobility_data.py") @@ -61,9 +61,6 @@ rule build_egon_data: rule prepare_district_heating_subnodes: - params: - district_heating=config_provider("sector", "district_heating"), - baseyear=config_provider("scenario", "planning_horizons", 0), input: heating_technologies_nuts3=resources("heating_technologies_nuts3.geojson"), regions_onshore=resources("regions_onshore_base_s_{clusters}.geojson"), @@ -95,6 +92,9 @@ rule prepare_district_heating_subnodes: ), resources: mem_mb=20000, + params: + district_heating=config_provider("sector", "district_heating"), + baseyear=config_provider("scenario", "planning_horizons", 0), script: scripts("pypsa-de/prepare_district_heating_subnodes.py") @@ -104,20 +104,6 @@ def baseyear_value(wildcards): rule add_district_heating_subnodes: - params: - district_heating=config_provider("sector", "district_heating"), - baseyear=config_provider("scenario", "planning_horizons", 0), - sector=config_provider("sector"), - heat_pump_sources=config_provider( - "sector", "heat_pump_sources", "urban central" - ), - heat_utilisation_potentials=config_provider( - "sector", "district_heating", "heat_utilisation_potentials" - ), - direct_utilisation_heat_sources=config_provider( - "sector", "district_heating", "direct_utilisation_heat_sources" - ), - adjustments=config_provider("adjustments", "sector"), input: unpack(input_heat_source_power), network=resources( @@ -152,6 +138,20 @@ rule add_district_heating_subnodes: ), resources: mem_mb=10000, + params: + district_heating=config_provider("sector", "district_heating"), + baseyear=config_provider("scenario", "planning_horizons", 0), + sector=config_provider("sector"), + heat_pump_sources=config_provider( + "sector", "heat_pump_sources", "urban central" + ), + heat_utilisation_potentials=config_provider( + "sector", "district_heating", "heat_utilisation_potentials" + ), + direct_utilisation_heat_sources=config_provider( + "sector", "district_heating", "direct_utilisation_heat_sources" + ), + adjustments=config_provider("adjustments", "sector"), script: scripts("pypsa-de/add_district_heating_subnodes.py") @@ -160,8 +160,6 @@ ruleorder: modify_district_heat_share > build_district_heat_share rule modify_district_heat_share: - params: - district_heating=config_provider("sector", "district_heating"), input: heating_technologies_nuts3=resources("heating_technologies_nuts3.geojson"), regions_onshore=resources("regions_onshore_base_s_{clusters}.geojson"), @@ -172,52 +170,17 @@ rule modify_district_heat_share: district_heat_share=resources( "district_heat_share_base_s_{clusters}_{planning_horizons}-modified.csv" ), - resources: - mem_mb=1000, log: logs("modify_district_heat_share_{clusters}_{planning_horizons}.log"), + resources: + mem_mb=1000, + params: + district_heating=config_provider("sector", "district_heating"), script: scripts("pypsa-de/modify_district_heat_share.py") rule modify_prenetwork: - params: - efuel_export_ban=config_provider("solving", "constraints", "efuel_export_ban"), - enable_kernnetz=config_provider("wasserstoff_kernnetz", "enable"), - technology_occurrence=config_provider("first_technology_occurrence"), - fossil_boiler_ban=config_provider("new_decentral_fossil_boiler_ban"), - coal_ban=config_provider("coal_generation_ban"), - nuclear_ban=config_provider("nuclear_generation_ban"), - planning_horizons=config_provider("scenario", "planning_horizons"), - H2_transmission_efficiency=config_provider( - "sector", "transmission_efficiency", "H2 pipeline" - ), - H2_retrofit=config_provider("sector", "H2_retrofit"), - H2_retrofit_capacity_per_CH4=config_provider( - "sector", "H2_retrofit_capacity_per_CH4" - ), - transmission_costs=config_provider("costs", "transmission"), - must_run=config_provider("must_run"), - clustering=config_provider("clustering", "temporal", "resolution_sector"), - H2_plants=config_provider("electricity", "H2_plants"), - onshore_nep_force=config_provider("onshore_nep_force"), - offshore_nep_force=config_provider("offshore_nep_force"), - shipping_methanol_efficiency=config_provider( - "sector", "shipping_methanol_efficiency" - ), - shipping_oil_efficiency=config_provider("sector", "shipping_oil_efficiency"), - shipping_methanol_share=config_provider("sector", "shipping_methanol_share"), - scale_capacity=config_provider("scale_capacity"), - bev_charge_rate=config_provider("sector", "bev_charge_rate"), - bev_energy=config_provider("sector", "bev_energy"), - bev_dsm_availability=config_provider("sector", "bev_dsm_availability"), - uba_for_industry=config_provider("pypsa-de", "uba_for_industry", "enable"), - scale_industry_non_energy=config_provider( - "pypsa-de", "uba_for_industry", "scale_industry_non_energy" - ), - limit_cross_border_flows_ac=config_provider( - "pypsa-de", "limit_cross_border_flows_ac" - ), input: network=resources( "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}_brownfield.nc" @@ -255,11 +218,48 @@ rule modify_prenetwork: network=resources( "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}_final.nc" ), - resources: - mem_mb=4000, log: RESULTS + "logs/modify_prenetwork_base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.log", + resources: + mem_mb=4000, + params: + efuel_export_ban=config_provider("solving", "constraints", "efuel_export_ban"), + enable_kernnetz=config_provider("wasserstoff_kernnetz", "enable"), + technology_occurrence=config_provider("first_technology_occurrence"), + fossil_boiler_ban=config_provider("new_decentral_fossil_boiler_ban"), + coal_ban=config_provider("coal_generation_ban"), + nuclear_ban=config_provider("nuclear_generation_ban"), + planning_horizons=config_provider("scenario", "planning_horizons"), + H2_transmission_efficiency=config_provider( + "sector", "transmission_efficiency", "H2 pipeline" + ), + H2_retrofit=config_provider("sector", "H2_retrofit"), + H2_retrofit_capacity_per_CH4=config_provider( + "sector", "H2_retrofit_capacity_per_CH4" + ), + transmission_costs=config_provider("costs", "transmission"), + must_run=config_provider("must_run"), + clustering=config_provider("clustering", "temporal", "resolution_sector"), + H2_plants=config_provider("electricity", "H2_plants"), + onshore_nep_force=config_provider("onshore_nep_force"), + offshore_nep_force=config_provider("offshore_nep_force"), + shipping_methanol_efficiency=config_provider( + "sector", "shipping_methanol_efficiency" + ), + shipping_oil_efficiency=config_provider("sector", "shipping_oil_efficiency"), + shipping_methanol_share=config_provider("sector", "shipping_methanol_share"), + scale_capacity=config_provider("scale_capacity"), + bev_charge_rate=config_provider("sector", "bev_charge_rate"), + bev_energy=config_provider("sector", "bev_energy"), + bev_dsm_availability=config_provider("sector", "bev_dsm_availability"), + uba_for_industry=config_provider("pypsa-de", "uba_for_industry", "enable"), + scale_industry_non_energy=config_provider( + "pypsa-de", "uba_for_industry", "scale_industry_non_energy" + ), + limit_cross_border_flows_ac=config_provider( + "pypsa-de", "limit_cross_border_flows_ac" + ), script: scripts("pypsa-de/modify_prenetwork.py") @@ -273,19 +273,15 @@ rule modify_existing_heating: existing_heating="data/existing_infrastructure/existing_heating_raw.csv", output: existing_heating=resources("existing_heating.csv"), - resources: - mem_mb=1000, log: logs("modify_existing_heating.log"), + resources: + mem_mb=1000, script: scripts("pypsa-de/modify_existing_heating.py") rule build_existing_chp_de: - params: - district_heating_subnodes=config_provider( - "sector", "district_heating", "subnodes" - ), input: mastr_biomass="data/mastr/bnetza_open_mastr_2023-08-08_B_biomass.csv", mastr_combustion="data/mastr/bnetza_open_mastr_2023-08-08_B_combustion.csv", @@ -301,17 +297,19 @@ rule build_existing_chp_de: ), output: german_chp=resources("german_chp_base_s_{clusters}.csv"), - resources: - mem_mb=4000, log: logs("build_existing_chp_de_{clusters}.log"), + resources: + mem_mb=4000, + params: + district_heating_subnodes=config_provider( + "sector", "district_heating", "subnodes" + ), script: scripts("pypsa-de/build_existing_chp_de.py") rule modify_industry_production: - params: - reference_scenario=config_provider("pypsa-de", "reference_scenario"), input: ariadne="data/ariadne_database.csv", industrial_production_per_country_tomorrow=resources( @@ -321,17 +319,17 @@ rule modify_industry_production: industrial_production_per_country_tomorrow=resources( "industrial_production_per_country_tomorrow_{planning_horizons}-modified.csv" ), - resources: - mem_mb=1000, log: logs("modify_industry_production_{planning_horizons}.log"), + resources: + mem_mb=1000, + params: + reference_scenario=config_provider("pypsa-de", "reference_scenario"), script: scripts("pypsa-de/modify_industry_production.py") rule build_wasserstoff_kernnetz: - params: - kernnetz=config_provider("wasserstoff_kernnetz"), input: wasserstoff_kernnetz_1=storage( "https://fnb-gas.de/wp-content/uploads/2024/07/2024_07_22_Anlage2_Leitungsmeldungen_weiterer_potenzieller_Wasserstoffnetzbetreiber.xlsx", @@ -356,13 +354,13 @@ rule build_wasserstoff_kernnetz: cleaned_wasserstoff_kernnetz=resources("wasserstoff_kernnetz.csv"), log: logs("build_wasserstoff_kernnetz.log"), + params: + kernnetz=config_provider("wasserstoff_kernnetz"), script: scripts("pypsa-de/build_wasserstoff_kernnetz.py") rule cluster_wasserstoff_kernnetz: - params: - kernnetz=config_provider("wasserstoff_kernnetz"), input: cleaned_h2_network=resources("wasserstoff_kernnetz.csv"), regions_onshore=resources("regions_onshore_base_s_{clusters}.geojson"), @@ -371,5 +369,7 @@ rule cluster_wasserstoff_kernnetz: clustered_h2_network=resources("wasserstoff_kernnetz_base_s_{clusters}.csv"), log: logs("cluster_wasserstoff_kernnetz_{clusters}.log"), + params: + kernnetz=config_provider("wasserstoff_kernnetz"), script: scripts("pypsa-de/cluster_wasserstoff_kernnetz.py") diff --git a/rules/pypsa-de/reporting.smk b/rules/pypsa-de/reporting.smk index 629955e4c..c3c546b8e 100644 --- a/rules/pypsa-de/reporting.smk +++ b/rules/pypsa-de/reporting.smk @@ -4,15 +4,6 @@ rule export_ariadne_variables: - params: - planning_horizons=config_provider("scenario", "planning_horizons"), - hours=config_provider("clustering", "temporal", "resolution_sector"), - config_industry=config_provider("industry"), - energy_totals_year=config_provider("energy", "energy_totals_year"), - co2_sequestration_cost=config_provider("sector", "co2_sequestration_cost"), - post_discretization=config_provider("solving", "options", "post_discretization"), - NEP_year=lambda w: config_provider("costs", "custom_cost_fn")(w)[-8:-4], - NEP_transmission=config_provider("costs", "transmission"), input: template="data/template_ariadne_database.xlsx", industry_demands=expand( @@ -50,17 +41,24 @@ rule export_ariadne_variables: output: exported_variables=RESULTS + "ariadne/exported_variables.xlsx", exported_variables_full=RESULTS + "ariadne/exported_variables_full.xlsx", - resources: - mem_mb=16000, log: RESULTS + "logs/export_ariadne_variables.log", + resources: + mem_mb=16000, + params: + planning_horizons=config_provider("scenario", "planning_horizons"), + hours=config_provider("clustering", "temporal", "resolution_sector"), + config_industry=config_provider("industry"), + energy_totals_year=config_provider("energy", "energy_totals_year"), + co2_sequestration_cost=config_provider("sector", "co2_sequestration_cost"), + post_discretization=config_provider("solving", "options", "post_discretization"), + NEP_year=lambda w: config_provider("costs", "custom_cost_fn")(w)[-8:-4], + NEP_transmission=config_provider("costs", "transmission"), script: scripts("pypsa-de/export_ariadne_variables.py") rule plot_ariadne_variables: - params: - reference_scenario=config_provider("pypsa-de", "reference_scenario"), input: exported_variables_full=RESULTS + "ariadne/exported_variables_full.xlsx", ariadne_database="data/ariadne_database.csv", @@ -99,14 +97,13 @@ rule plot_ariadne_variables: trade_balance=RESULTS + "ariadne/trade-balance-DE.pdf", log: RESULTS + "logs/plot_ariadne_variables.log", + params: + reference_scenario=config_provider("pypsa-de", "reference_scenario"), script: scripts("pypsa-de/plot_ariadne_variables.py") rule plot_hydrogen_network_incl_kernnetz: - params: - plotting=config_provider("plotting"), - foresight=config_provider("foresight"), input: network=RESULTS + "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc", @@ -114,9 +111,6 @@ rule plot_hydrogen_network_incl_kernnetz: output: map=RESULTS + "maps/base_s_{clusters}_{opts}_{sector_opts}-h2_network_incl_kernnetz_{planning_horizons}.pdf", - threads: 2 - resources: - mem_mb=10000, log: RESULTS + "logs/plot_hydrogen_network_incl_kernnetz/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.log", @@ -125,20 +119,17 @@ rule plot_hydrogen_network_incl_kernnetz: RESULTS + "benchmarks/plot_hydrogen_network_incl_kernnetz/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}" ) + threads: 2 + resources: + mem_mb=10000, + params: + plotting=config_provider("plotting"), + foresight=config_provider("foresight"), script: scripts("pypsa-de/plot_hydrogen_network_incl_kernnetz.py") rule plot_ariadne_report: - params: - planning_horizons=config_provider("scenario", "planning_horizons"), - plotting=config_provider("plotting"), - run=config_provider("run", "name"), - foresight=config_provider("foresight"), - post_discretization=config_provider("solving", "options", "post_discretization"), - NEP_year=lambda w: config_provider("costs", "custom_cost_fn")(w)[-8:-4], - hours=config_provider("clustering", "temporal", "resolution_sector"), - NEP_transmission=config_provider("costs", "transmission"), input: networks=expand( RESULTS @@ -171,10 +162,19 @@ rule plot_ariadne_report: elec_balances=directory(RESULTS + "ariadne/report/elec_balance_timeseries"), heat_balances=directory(RESULTS + "ariadne/report/heat_balance_timeseries"), nodal_balances=directory(RESULTS + "ariadne/report/balance_timeseries_2045"), - resources: - mem_mb=32000, log: RESULTS + "logs/plot_ariadne_report.log", + resources: + mem_mb=32000, + params: + planning_horizons=config_provider("scenario", "planning_horizons"), + plotting=config_provider("plotting"), + run=config_provider("run", "name"), + foresight=config_provider("foresight"), + post_discretization=config_provider("solving", "options", "post_discretization"), + NEP_year=lambda w: config_provider("costs", "custom_cost_fn")(w)[-8:-4], + hours=config_provider("clustering", "temporal", "resolution_sector"), + NEP_transmission=config_provider("costs", "transmission"), script: scripts("pypsa-de/plot_ariadne_report.py") diff --git a/rules/pypsa-de/retrieve.smk b/rules/pypsa-de/retrieve.smk index 0a523c12e..dde51bae6 100644 --- a/rules/pypsa-de/retrieve.smk +++ b/rules/pypsa-de/retrieve.smk @@ -8,14 +8,14 @@ if config["pypsa-de"]["use_internal_db"]: ] in ["primary"]: rule retrieve_ariadne_database_internal: - params: - source="internal", output: data="data/ariadne_database.csv", log: "logs/retrieve_ariadne_database_internal_primary.log", resources: mem_mb=1000, + params: + source="internal", script: scripts("pypsa-de/retrieve_ariadne_database.py") @@ -25,14 +25,14 @@ else: ]: rule retrieve_ariadne_database: - params: - source="primary", output: data="data/ariadne_database.csv", log: "logs/retrieve_ariadne_database_primary.log", resources: mem_mb=1000, + params: + source="primary", script: scripts("pypsa-de/retrieve_ariadne_database.py") @@ -41,9 +41,6 @@ else: ]: rule retrieve_ariadne_database: - params: - source="archive", - version=ARIADNE_DATABASE["version"], input: raw_xlsx=storage(ARIADNE_DATABASE["url"]), output: @@ -52,6 +49,9 @@ else: "logs/retrieve_ariadne_database_archive.log", resources: mem_mb=1000, + params: + source="archive", + version=ARIADNE_DATABASE["version"], script: scripts("pypsa-de/retrieve_ariadne_database.py") @@ -75,11 +75,11 @@ if (OPEN_MASTR := dataset_version("open_mastr"))["source"] in ["primary", "archi rule retrieve_open_mastr: input: storage(OPEN_MASTR["url"]), - params: - "data/mastr", output: "data/mastr/bnetza_open_mastr_2023-08-08_B_biomass.csv", "data/mastr/bnetza_open_mastr_2023-08-08_B_combustion.csv", + params: + "data/mastr", run: unpack_archive(input[0], params[0]) @@ -87,12 +87,12 @@ if (OPEN_MASTR := dataset_version("open_mastr"))["source"] in ["primary", "archi if (EGON := dataset_version("egon"))["source"] in ["build"]: rule retrieve_egon_data: - params: - url=EGON["url"], - folder=EGON["folder"], output: spatial=f"{EGON['folder']}/demandregio_spatial_2018.json", mapping=f"{EGON['folder']}/mapping_technologies.json", + params: + url=EGON["url"], + folder=EGON["folder"], shell: """ mkdir -p {params.folder} diff --git a/rules/retrieve.smk b/rules/retrieve.smk index dbb9c8662..cde7f3215 100755 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -28,12 +28,12 @@ if (EUROSTAT_BALANCES_DATASET := dataset_version("eurostat_balances"))["source"] ]: rule retrieve_eurostat_balances: - message: - "Retrieving Eurostat balances data" input: tsv_gz=storage(EUROSTAT_BALANCES_DATASET["url"]), output: tsv_gz=f"{EUROSTAT_BALANCES_DATASET['folder']}/estat_nrg_bal_c.tsv.gz", + message: + "Retrieving Eurostat balances data" run: copy2(input["tsv_gz"], output["tsv_gz"]) @@ -48,12 +48,12 @@ if ( ]: rule retrieve_eurostat_household_balances: - message: - "Retrieving Eurostat household balances data" input: csv=storage(EUROSTAT_HOUSEHOLD_BALANCES_DATASET["url"]), output: csv=f"{EUROSTAT_HOUSEHOLD_BALANCES_DATASET['folder']}/nrg_d_hhq.csv", + message: + "Retrieving Eurostat household balances data" run: copy2(input["csv"], output["csv"]) @@ -66,12 +66,12 @@ if (SWISS_ENERGY_BALANCES_DATASET := dataset_version("swiss_energy_balances"))[ ]: rule retrieve_swiss_energy_balances: - message: - "Retrieving Swiss energy balances data" input: xlsx=storage(SWISS_ENERGY_BALANCES_DATASET["url"]), output: xlsx=f"{SWISS_ENERGY_BALANCES_DATASET['folder']}/12361-VWZ_Webtabellen_2024.xlsx", + message: + "Retrieving Swiss energy balances data" run: copy2(input["xlsx"], output["xlsx"]) @@ -82,13 +82,13 @@ if (NUTS3_POPULATION_DATASET := dataset_version("nuts3_population"))["source"] i ]: rule retrieve_nuts3_population: - message: - "Retrieving NUTS3 population data" input: gz=storage(NUTS3_POPULATION_DATASET["url"]), output: gz=f"{NUTS3_POPULATION_DATASET['folder']}/nama_10r_3popgdp.tsv.gz", retries: 2 + message: + "Retrieving NUTS3 population data" run: copy2(input["gz"], output["gz"]) @@ -96,13 +96,13 @@ if (NUTS3_POPULATION_DATASET := dataset_version("nuts3_population"))["source"] i if (CORINE_DATASET := dataset_version("corine"))["source"] in ["archive"]: rule retrieve_corine: - message: - "Retrieving Corine land cover data" input: zip_file=storage(CORINE_DATASET["url"]), output: zip_file=f"{CORINE_DATASET['folder']}/corine.zip", tif_file=f"{CORINE_DATASET['folder']}/corine.tif", + message: + "Retrieving Corine land cover data" run: output_folder = Path(output["zip_file"]).parent unpack_archive(input["zip_file"], output_folder) @@ -114,18 +114,18 @@ if (CORINE_DATASET := dataset_version("corine"))["source"] in ["archive"]: elif (CORINE_DATASET := dataset_version("corine"))["source"] in ["primary"]: rule retrieve_corine: - message: - "Retrieving Corine land cover data" - params: - apikey=os.environ.get("CORINE_API_TOKEN", ""), output: zip=f"{CORINE_DATASET['folder']}/corine.zip", tif_file=f"{CORINE_DATASET['folder']}/corine.tif", log: logs("retrieve_corine_primary.log"), + retries: 2 resources: mem_mb=1000, - retries: 2 + params: + apikey=os.environ.get("CORINE_API_TOKEN", ""), + message: + "Retrieving Corine land cover data" script: scripts("retrieve_corine_dataset_primary.py") @@ -135,13 +135,13 @@ if (H2_SALT_CAVERNS_DATASET := dataset_version("h2_salt_caverns"))["source"] in ]: rule retrieve_h2_salt_caverns: - message: - "Retrieving H2 salt caverns data" input: geojson=storage(H2_SALT_CAVERNS_DATASET["url"]), output: geojson=f"{H2_SALT_CAVERNS_DATASET['folder']}/h2_salt_caverns_GWh_per_sqkm.geojson", retries: 2 + message: + "Retrieving H2 salt caverns data" run: copy2(input["geojson"], output["geojson"]) @@ -151,13 +151,13 @@ if (GDP_PER_CAPITA_DATASET := dataset_version("gdp_per_capita"))["source"] in [ ]: rule retrieve_gdp_per_capita: - message: - "Retrieving GDP per capita data" input: gdp=storage(GDP_PER_CAPITA_DATASET["url"]), output: gdp=f"{GDP_PER_CAPITA_DATASET['folder']}/GDP_per_capita_PPP_1990_2015_v2.nc", retries: 2 + message: + "Retrieving GDP per capita data" run: copy2(input["gdp"], output["gdp"]) @@ -168,13 +168,13 @@ if (POPULATION_COUNT_DATASET := dataset_version("population_count"))["source"] i ]: rule retrieve_population_count: - message: - "Retrieving population count data" input: tif=storage(POPULATION_COUNT_DATASET["url"]), output: tif=f"{POPULATION_COUNT_DATASET['folder']}/ppp_2019_1km_Aggregated.tif", retries: 2 + message: + "Retrieving population count data" run: copy2(input["tif"], output["tif"]) @@ -195,8 +195,6 @@ if (GHG_EMISSIONS_DATASET := dataset_version("ghg_emissions"))["source"] in [ ]: rule retrieve_ghg_emissions: - message: - "Retrieving GHG emissions data" input: ghg=storage(GHG_EMISSIONS_DATASET["url"]), output: @@ -212,6 +210,8 @@ if (GHG_EMISSIONS_DATASET := dataset_version("ghg_emissions"))["source"] in [ else [] ), retries: 2 + message: + "Retrieving GHG emissions data" run: if GHG_EMISSIONS_DATASET["source"] == "primary": copy2(input["ghg"], output["zip"]) @@ -224,8 +224,6 @@ if (GHG_EMISSIONS_DATASET := dataset_version("ghg_emissions"))["source"] in [ if (GEBCO_DATASET := dataset_version("gebco"))["source"] in ["archive", "primary"]: rule retrieve_gebco: - message: - "Retrieving GEBCO bathymetry data" input: storage(GEBCO_DATASET["url"]), output: @@ -235,6 +233,8 @@ if (GEBCO_DATASET := dataset_version("gebco"))["source"] in ["archive", "primary if GEBCO_DATASET["source"] == "primary" else [] ), + message: + "Retrieving GEBCO bathymetry data" run: if GEBCO_DATASET["source"] == "primary": import xarray as xr @@ -259,13 +259,13 @@ if (ATTRIBUTED_PORTS_DATASET := dataset_version("attributed_ports"))["source"] i ]: rule retrieve_attributed_ports: - message: - "Retrieving attributed ports data" input: json=storage(ATTRIBUTED_PORTS_DATASET["url"]), output: json=f"{ATTRIBUTED_PORTS_DATASET['folder']}/attributed_ports.json", retries: 2 + message: + "Retrieving attributed ports data" run: copy2(input["json"], output["json"]) @@ -276,13 +276,13 @@ if (JRC_IDEES_DATASET := dataset_version("jrc_idees"))["source"] in [ ]: rule retrieve_jrc_idees: - message: - "Retrieving JRC IDEES data" input: zip_file=storage(JRC_IDEES_DATASET["url"]), output: zip_file=f"{JRC_IDEES_DATASET['folder']}/jrc_idees.zip", directory=directory(JRC_IDEES_DATASET["folder"]), + message: + "Retrieving JRC IDEES data" run: copy2(input["zip_file"], output["zip_file"]) output_folder = Path(output["zip_file"]).parent @@ -295,8 +295,6 @@ if (EU_NUTS2013_DATASET := dataset_version("eu_nuts2013"))["source"] in [ ]: rule retrieve_eu_nuts_2013: - message: - "Retrieving EU NUTS 2013 data" input: shapes=storage(EU_NUTS2013_DATASET["url"]), output: @@ -306,6 +304,8 @@ if (EU_NUTS2013_DATASET := dataset_version("eu_nuts2013"))["source"] in [ ), shapes_level_3=f"{EU_NUTS2013_DATASET['folder']}/ref-nuts-2013-03m.geojson/NUTS_RG_03M_2013_4326_LEVL_3.geojson", shapes_level_2=f"{EU_NUTS2013_DATASET['folder']}/ref-nuts-2013-03m.geojson/NUTS_RG_03M_2013_4326_LEVL_2.geojson", + message: + "Retrieving EU NUTS 2013 data" run: copy2(input["shapes"], output["zip_file"]) unpack_archive(output["zip_file"], Path(output.shapes_level_3).parent) @@ -317,8 +317,6 @@ if (EU_NUTS2021_DATASET := dataset_version("eu_nuts2021"))["source"] in [ ]: rule retrieve_eu_nuts_2021: - message: - "Retrieving EU NUTS 2021 data" input: shapes=storage(EU_NUTS2021_DATASET["url"]), output: @@ -330,6 +328,8 @@ if (EU_NUTS2021_DATASET := dataset_version("eu_nuts2021"))["source"] in [ shapes_level_2=f"{EU_NUTS2021_DATASET['folder']}/ref-nuts-2021-01m.geojson/NUTS_RG_01M_2021_4326_LEVL_2.geojson", shapes_level_1=f"{EU_NUTS2021_DATASET['folder']}/ref-nuts-2021-01m.geojson/NUTS_RG_01M_2021_4326_LEVL_1.geojson", shapes_level_0=f"{EU_NUTS2021_DATASET['folder']}/ref-nuts-2021-01m.geojson/NUTS_RG_01M_2021_4326_LEVL_0.geojson", + message: + "Retrieving EU NUTS 2021 data" run: copy2(input["shapes"], output["zip_file"]) unpack_archive(output["zip_file"], Path(output.shapes_level_3).parent) @@ -348,9 +348,9 @@ if ( geojson=f"{BIDDING_ZONES_ELECTRICITYMAPS_DATASET['folder']}/bidding_zones_electricitymaps.geojson", log: "logs/retrieve_bidding_zones_electricitymaps.log", + retries: 2 resources: mem_mb=1000, - retries: 2 run: copy2(input["geojson"], output["geojson"]) @@ -364,9 +364,9 @@ if (BIDDING_ZONES_ENTSOEPY_DATASET := dataset_version("bidding_zones_entsoepy")) geojson=f"{BIDDING_ZONES_ENTSOEPY_DATASET['folder']}/bidding_zones_entsoepy.geojson", log: "logs/retrieve_bidding_zones_entsoepy.log", + retries: 2 resources: mem_mb=1000, - retries: 2 run: import entsoe import geopandas as gpd @@ -398,17 +398,17 @@ if (CUTOUT_DATASET := dataset_version("cutout"))["source"] in [ ]: rule retrieve_cutout: - message: - "Retrieving cutout data for {wildcards.cutout}" input: storage(CUTOUT_DATASET["url"] + "/{cutout}.nc"), output: CUTOUT_DATASET["folder"] + "/{cutout}.nc", log: "logs/retrieve_cutout/{cutout}.log", + retries: 2 resources: mem_mb=5000, - retries: 2 + message: + "Retrieving cutout data for {wildcards.cutout}" run: copy2(input[0], output[0]) @@ -418,12 +418,12 @@ if (COUNTRY_RUNOFF_DATASET := dataset_version("country_runoff"))["source"] in [ ]: rule retrieve_country_runoff: - message: - "Retrieving country runoff data" input: storage(COUNTRY_RUNOFF_DATASET["url"]), output: era5_runoff=f"{COUNTRY_RUNOFF_DATASET['folder']}/era5-runoff-per-country.csv", + message: + "Retrieving country runoff data" run: copy2(input[0], output[0]) @@ -431,12 +431,12 @@ if (COUNTRY_RUNOFF_DATASET := dataset_version("country_runoff"))["source"] in [ if (COUNTRY_HDD_DATASET := dataset_version("country_hdd"))["source"] in ["archive"]: rule retrieve_country_hdd: - message: - "Retrieving country heating degree days data" input: storage(COUNTRY_HDD_DATASET["url"]), output: era5_runoff=f"{COUNTRY_HDD_DATASET['folder']}/era5-HDD-per-country.csv", + message: + "Retrieving country heating degree days data" run: copy2(input[0], output[0]) @@ -447,12 +447,12 @@ if (COSTS_DATASET := dataset_version("costs"))["source"] in [ ]: rule retrieve_cost_data: - message: - "Retrieving cost data for {wildcards.planning_horizons}" input: costs=storage(COSTS_DATASET["url"] + "/costs_{planning_horizons}.csv"), output: costs=COSTS_DATASET["folder"] + "/costs_{planning_horizons}.csv", + message: + "Retrieving cost data for {wildcards.planning_horizons}" run: copy2(input["costs"], output["costs"]) @@ -463,12 +463,12 @@ if (POWERPLANTS_DATASET := dataset_version("powerplants"))["source"] in [ ]: rule retrieve_powerplants: - message: - "Retrieving powerplants data" input: powerplants=storage(POWERPLANTS_DATASET["url"]), output: powerplants=f"{POWERPLANTS_DATASET['folder']}/powerplants.csv", + message: + "Retrieving powerplants data" run: copy2(input["powerplants"], output["powerplants"]) @@ -479,8 +479,6 @@ if (SCIGRID_GAS_DATASET := dataset_version("scigrid_gas"))["source"] in [ ]: rule retrieve_gas_infrastructure_data: - message: - "Retrieving SciGRID gas infrastructure data" input: zip_file=storage(SCIGRID_GAS_DATASET["url"]), output: @@ -488,6 +486,8 @@ if (SCIGRID_GAS_DATASET := dataset_version("scigrid_gas"))["source"] in [ entry=f"{SCIGRID_GAS_DATASET['folder']}/data/IGGIELGN_BorderPoints.geojson", storage=f"{SCIGRID_GAS_DATASET['folder']}/data/IGGIELGN_Storages.geojson", gas_network=f"{SCIGRID_GAS_DATASET['folder']}/data/IGGIELGN_PipeSegments.geojson", + message: + "Retrieving SciGRID gas infrastructure data" run: copy2(input["zip_file"], output["zip_file"]) output_folder = Path(output["zip_file"]).parent @@ -499,17 +499,17 @@ if (OPSD_DEMAND_DATA := dataset_version("opsd_electricity_demand"))["source"] in ]: rule retrieve_electricity_demand_opsd: - message: - "Retrieving electricity demand data from OPSD from build source" - params: - versions=["2019-06-05", "2020-10-06"], output: csv=f"{OPSD_DEMAND_DATA['folder']}/electricity_demand_opsd_raw.csv", log: "logs/retrieve_electricity_demand_opsd.log", + retries: 2 resources: mem_mb=5000, - retries: 2 + params: + versions=["2019-06-05", "2020-10-06"], + message: + "Retrieving electricity demand data from OPSD from build source" script: scripts("retrieve_electricity_demand_opsd.py") @@ -519,13 +519,13 @@ if (OPSD_DEMAND_DATA := dataset_version("opsd_electricity_demand"))["source"] in ]: rule retrieve_electricity_demand_opsd: - message: - "Retrieving electricity demand data from OPSD from archive" input: csv=storage(OPSD_DEMAND_DATA["url"]), output: csv=f"{OPSD_DEMAND_DATA['folder']}/electricity_demand_opsd_raw.csv", retries: 2 + message: + "Retrieving electricity demand data from OPSD from archive" run: copy2(input["csv"], output["csv"]) @@ -575,24 +575,22 @@ if (ENTSOE_DEMAND_DATA := dataset_version("entsoe_electricity_demand"))["source" ] rule retrieve_electricity_demand_entsoe_country: - message: - "Retrieving electricity demand data from ENTSO-E for {wildcards.country}" - params: - entsoe_token=os.environ.get("ENTSOE_API_TOKEN", ""), output: csv=f"{ENTSOE_DEMAND_DATA['folder']}" + "/electricity_demand_entsoe_raw_{country}.csv", log: "logs/retrieve_electricity_demand_entsoe_{country}.log", + retries: 2 resources: mem_mb=2000, - retries: 2 + params: + entsoe_token=os.environ.get("ENTSOE_API_TOKEN", ""), + message: + "Retrieving electricity demand data from ENTSO-E for {wildcards.country}" script: scripts("retrieve_electricity_demand_entsoe.py") rule retrieve_electricity_demand_entsoe: - message: - "Retrieving electricity demand data from ENTSO-E from build source" input: csvs=expand( f"{ENTSOE_DEMAND_DATA['folder']}" @@ -601,6 +599,8 @@ if (ENTSOE_DEMAND_DATA := dataset_version("entsoe_electricity_demand"))["source" ), output: csv=f"{ENTSOE_DEMAND_DATA['folder']}/electricity_demand_entsoe_raw.csv", + message: + "Retrieving electricity demand data from ENTSO-E from build source" run: import pandas as pd @@ -614,13 +614,13 @@ if (ENTSOE_DEMAND_DATA := dataset_version("entsoe_electricity_demand"))["source" ]: rule retrieve_electricity_demand_entsoe: - message: - "Retrieving electricity demand data from ENTSO-E from archive" input: csv=storage(ENTSOE_DEMAND_DATA["url"]), output: csv=f"{ENTSOE_DEMAND_DATA['folder']}/electricity_demand_entsoe_raw.csv", retries: 2 + message: + "Retrieving electricity demand data from ENTSO-E from archive" run: copy2(input["csv"], output["csv"]) @@ -630,15 +630,15 @@ if (NESO_DEMAND_DATA := dataset_version("neso_electricity_demand"))["source"] in ]: rule retrieve_electricity_demand_neso: - message: - "Retrieving electricity demand data from NESO from build source" output: csv=f"{NESO_DEMAND_DATA['folder']}/electricity_demand_neso_raw.csv", log: "logs/retrieve_electricity_demand_neso.log", + retries: 2 resources: mem_mb=5000, - retries: 2 + message: + "Retrieving electricity demand data from NESO from build source" script: scripts("retrieve_electricity_demand_neso.py") @@ -648,13 +648,13 @@ if (NESO_DEMAND_DATA := dataset_version("neso_electricity_demand"))["source"] in ]: rule retrieve_electricity_demand_neso: - message: - "Retrieving electricity demand data from NESO from archive" input: csv=storage(NESO_DEMAND_DATA["url"]), output: csv=f"{NESO_DEMAND_DATA['folder']}/electricity_demand_neso_raw.csv", retries: 2 + message: + "Retrieving electricity demand data from NESO from archive" run: copy2(input["csv"], output["csv"]) @@ -669,13 +669,13 @@ if ( ]: rule retrieve_synthetic_electricity_demand: - message: - "Retrieving synthetic electricity demand data" input: csv=storage(SYNTHETIC_ELECTRICITY_DEMAND_DATASET["url"]), output: csv=f"{SYNTHETIC_ELECTRICITY_DEMAND_DATASET['folder']}/load_synthetic_raw.csv", retries: 2 + message: + "Retrieving synthetic electricity demand data" run: copy2(input["csv"], output["csv"]) @@ -686,10 +686,10 @@ if (ENERGY_ATLAS_DATASET := dataset_version("jrc_energy_atlas"))["source"] in [ ]: rule retrieve_electricity_demand_energy_atlas: - message: - "Retrieving JRC Energy Atlas electricity demand data raster" output: tif=f"{ENERGY_ATLAS_DATASET['folder']}/electricity_tot_demand_2019.tif", + message: + "Retrieving JRC Energy Atlas electricity demand data raster" run: import requests @@ -708,10 +708,10 @@ if ( )["source"] in ["primary", "archive"]: rule retrieve_desnz_electricity_consumption: - message: - "Retrieving DESNZ subnational electricity consumption data" output: xlsx=f"{DESNZ_ELECTRICITY_CONSUMPTION_DATASET['folder']}/Subnational_electricity_consumption_statistics_2005-2024.xlsx", + message: + "Retrieving DESNZ subnational electricity consumption data" run: import requests @@ -726,22 +726,22 @@ if ( if (ONS_LAD_DATASET := dataset_version("ons_lad"))["source"] in ["archive"]: rule retrieve_ons_lad: - message: - "Retrieving UK ONS Local Authority Districts (LAD) Boundaries data" input: geojson=storage(ONS_LAD_DATASET["url"]), output: geojson=f"{ONS_LAD_DATASET['folder']}/Local_Authority_Districts_May_2024_Boundaries__UK_BSC.geojson", + message: + "Retrieving UK ONS Local Authority Districts (LAD) Boundaries data" run: copy2(input["geojson"], output["geojson"]) elif ONS_LAD_DATASET["source"] in ["primary"]: rule retrieve_ons_lad: - message: - "Retrieving UK ONS Local Authority Districts (LAD) Boundaries data" output: geojson=f"{ONS_LAD_DATASET['folder']}/Local_Authority_Districts_May_2024_Boundaries__UK_BSC.geojson", + message: + "Retrieving UK ONS Local Authority Districts (LAD) Boundaries data" run: import requests @@ -763,17 +763,17 @@ if (SHIP_RASTER_DATASET := dataset_version("ship_raster"))["source"] in [ ]: rule retrieve_ship_raster: - message: - "Retrieving shipping raster data" input: zip_file=storage(SHIP_RASTER_DATASET["url"]), output: zip_file=f"{SHIP_RASTER_DATASET['folder']}/shipdensity_global.zip", log: "logs/retrieve_ship_raster.log", + retries: 2 resources: mem_mb=5000, - retries: 2 + message: + "Retrieving shipping raster data" run: copy2(input["zip_file"], output["zip_file"]) @@ -784,13 +784,13 @@ if (ENSPRESO_BIOMASS_DATASET := dataset_version("enspreso_biomass"))["source"] i ]: rule retrieve_enspreso_biomass: - message: - "Retrieving ENSPRESO biomass data" input: xlsx=storage(ENSPRESO_BIOMASS_DATASET["url"]), output: xlsx=f"{ENSPRESO_BIOMASS_DATASET['folder']}/ENSPRESO_BIOMASS.xlsx", retries: 1 + message: + "Retrieving ENSPRESO biomass data" run: copy2(input["xlsx"], output["xlsx"]) @@ -803,13 +803,13 @@ if (HOTMAPS_INDUSTRIAL_SITES := dataset_version("hotmaps_industrial_sites"))[ ]: rule retrieve_hotmaps_industrial_sites: - message: - "Retrieving Hotmaps industrial sites" input: csv=storage(HOTMAPS_INDUSTRIAL_SITES["url"]), output: csv=f"{HOTMAPS_INDUSTRIAL_SITES['folder']}/Industrial_Database.csv", retries: 1 + message: + "Retrieving Hotmaps industrial sites" run: copy2(input["csv"], output["csv"]) @@ -822,13 +822,13 @@ if (NITROGEN_STATISTICS_DATASET := dataset_version("nitrogen_statistics"))[ ]: rule retrieve_nitrogen_statistics: - message: - "Retrieving nitrogen statistics data" input: xlsx=storage(NITROGEN_STATISTICS_DATASET["url"]), output: xlsx=f"{NITROGEN_STATISTICS_DATASET['folder']}/nitro-ert.xlsx", retries: 1 + message: + "Retrieving nitrogen statistics data" run: copy2(input["xlsx"], output["xlsx"]) @@ -840,12 +840,12 @@ if (COPERNICUS_LAND_COVER_DATASET := dataset_version("copernicus_land_cover"))[ # Downloading Copernicus Global Land Cover for land cover and land use: # Website: https://land.copernicus.eu/global/products/lc rule download_copernicus_land_cover: - message: - "Retrieving Copernicus land cover data" input: tif=storage(COPERNICUS_LAND_COVER_DATASET["url"]), output: tif=f"{COPERNICUS_LAND_COVER_DATASET['folder']}/Copernicus_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif", + message: + "Retrieving Copernicus land cover data" run: copy2(input["tif"], output["tif"]) @@ -858,12 +858,12 @@ if (LUISA_LAND_COVER_DATASET := dataset_version("luisa_land_cover"))["source"] i # Downloading LUISA Base Map for land cover and land use: # Website: https://ec.europa.eu/jrc/en/luisa rule retrieve_luisa_land_cover: - message: - "Retrieving LUISA land cover data" input: tif=storage(LUISA_LAND_COVER_DATASET["url"]), output: tif=f"{LUISA_LAND_COVER_DATASET['folder']}/LUISA_basemap_020321_50m.tif", + message: + "Retrieving LUISA land cover data" run: copy2(input["tif"], output["tif"]) @@ -871,11 +871,11 @@ if (LUISA_LAND_COVER_DATASET := dataset_version("luisa_land_cover"))["source"] i if (EEZ_DATASET := dataset_version("eez"))["source"] in ["primary"]: rule retrieve_eez: - message: - "Retrieving EEZ data" output: zip_file=f"{EEZ_DATASET['folder']}/World_EEZ_{EEZ_DATASET['version']}_LR.zip", gpkg=f"{EEZ_DATASET['folder']}/World_EEZ_{EEZ_DATASET['version']}_LR/eez_{EEZ_DATASET['version'].split('_')[0]}_lowres.gpkg", + message: + "Retrieving EEZ data" run: from uuid import uuid4 @@ -905,8 +905,6 @@ if (EEZ_DATASET := dataset_version("eez"))["source"] in ["primary"]: elif (EEZ_DATASET := dataset_version("eez"))["source"] in ["archive"]: rule retrieve_eez: - message: - "Retrieving EEZ data" input: zip_file=storage( EEZ_DATASET["url"], @@ -914,6 +912,8 @@ elif (EEZ_DATASET := dataset_version("eez"))["source"] in ["archive"]: output: zip_file=f"{EEZ_DATASET['folder']}/World_EEZ_{EEZ_DATASET['version']}_LR.zip", gpkg=f"{EEZ_DATASET['folder']}/World_EEZ_{EEZ_DATASET['version']}_LR/eez_{EEZ_DATASET['version'].split('_')[0]}_lowres.gpkg", + message: + "Retrieving EEZ data" run: output_folder = Path(output["zip_file"]).parent copy2(input["zip_file"], output["zip_file"]) @@ -926,13 +926,13 @@ if (WB_URB_POP_DATASET := dataset_version("worldbank_urban_population"))["source ]: rule retrieve_worldbank_urban_population: - message: - "Retrieving World Bank urban population data" input: zip=storage(WB_URB_POP_DATASET["url"]), output: zip=f"{WB_URB_POP_DATASET['folder']}/API_SP.URB.TOTL.IN.ZS_DS2_en_csv_v2.zip", csv=f"{WB_URB_POP_DATASET['folder']}/API_SP.URB.TOTL.IN.ZS_DS2_en_csv_v2.csv", + message: + "Retrieving World Bank urban population data" run: copy2(input["zip"], output["zip"]) unpack_archive(output["zip"], WB_URB_POP_DATASET["folder"]) @@ -954,8 +954,6 @@ if (CO2STOP_DATASET := dataset_version("co2stop"))["source"] in [ ]: rule retrieve_co2stop: - message: - "Retrieving CO2STOP data" input: zip_file=storage(CO2STOP_DATASET["url"]), output: @@ -966,6 +964,8 @@ if (CO2STOP_DATASET := dataset_version("co2stop"))["source"] in [ traps_table2=f"{CO2STOP_DATASET['folder']}/CO2JRC_OpenFormats/CO2Stop_DataInterrogationSystem/Hydrocarbon_Traps_Temp.csv", traps_table3=f"{CO2STOP_DATASET['folder']}/CO2JRC_OpenFormats/CO2Stop_DataInterrogationSystem/Hydrocarbon_Traps1.csv", traps_map=f"{CO2STOP_DATASET['folder']}/CO2JRC_OpenFormats/CO2Stop_Polygons Data/DaughterUnits_March13.kml", + message: + "Retrieving CO2STOP data" run: output_folder = Path(output["zip_file"]).parent output_folder.mkdir(parents=True, exist_ok=True) @@ -981,12 +981,12 @@ if (GEM_EUROPE_GAS_TRACKER_DATASET := dataset_version("gem_europe_gas_tracker")) ]: rule retrieve_gem_europe_gas_tracker: - message: - "Retrieving GEM Europe Gas Tracker data" input: xlsx=storage(GEM_EUROPE_GAS_TRACKER_DATASET["url"]), output: xlsx="data/gem/Europe-Gas-Tracker-2024-05.xlsx", + message: + "Retrieving GEM Europe Gas Tracker data" run: copy2(input["xlsx"], output["xlsx"]) @@ -997,12 +997,12 @@ if (GEM_GSPT_DATASET := dataset_version("gem_gspt"))["source"] in [ ]: rule retrieve_gem_steel_plant_tracker: - message: - "Retrieving GEM Global Steel Plant Tracker data" input: xlsx=storage(GEM_GSPT_DATASET["url"]), output: xlsx=f"{GEM_GSPT_DATASET['folder']}/Global-Steel-Plant-Tracker.xlsx", + message: + "Retrieving GEM Global Steel Plant Tracker data" run: copy2(input["xlsx"], output["xlsx"]) @@ -1029,12 +1029,12 @@ if (BFS_ROAD_VEHICLE_STOCK_DATASET := dataset_version("bfs_road_vehicle_stock")) ]: rule retrieve_bfs_road_vehicle_stock: - message: - "Retrieving BFS road vehicle stock data" input: csv=storage(BFS_ROAD_VEHICLE_STOCK_DATASET["url"]), output: csv=f"{BFS_ROAD_VEHICLE_STOCK_DATASET['folder']}/vehicle_stock.csv", + message: + "Retrieving BFS road vehicle stock data" run: copy2(input["csv"], output["csv"]) @@ -1047,12 +1047,12 @@ if (BFS_GDP_AND_POPULATION_DATASET := dataset_version("bfs_gdp_and_population")) ]: rule retrieve_bfs_gdp_and_population: - message: - "Retrieving BFS GDP and population data" input: xlsx=storage(BFS_GDP_AND_POPULATION_DATASET["url"]), output: xlsx=f"{BFS_GDP_AND_POPULATION_DATASET['folder']}/gdp_and_population.xlsx", + message: + "Retrieving BFS GDP and population data" run: copy2(input["xlsx"], output["xlsx"]) @@ -1104,13 +1104,13 @@ if (WDPA_DATASET := dataset_version("wdpa"))["source"] in [ # extract the main zip and then merge the contained 3 zipped shapefiles # Website: https://www.protectedplanet.net/en/thematic-areas/wdpa rule retrieve_wdpa: - message: - "Downloading protected area database from WDPA" input: zip_file=storage(get_wdpa_url(WDPA_DATASET)), output: zip_file=f"{WDPA_DATASET['folder']}/WDPA_shp.zip", gpkg=f"{WDPA_DATASET['folder']}/WDPA.gpkg", + message: + "Downloading protected area database from WDPA" run: output_folder = Path(output["zip_file"]).parent copy2(input["zip_file"], output["zip_file"]) @@ -1138,16 +1138,16 @@ if (WDPA_MARINE_DATASET := dataset_version("wdpa_marine"))["source"] in [ ]: rule retrieve_wdpa_marine: - # Downloading Marine protected area database from WDPA - # extract the main zip and then merge the contained 3 zipped shapefiles - # Website: https://www.protectedplanet.net/en/thematic-areas/marine-protected-areas - message: - "Downloading Marine protected area database from WDPA" input: zip_file=storage(get_wdpa_url(WDPA_MARINE_DATASET)), output: zip_file=f"{WDPA_MARINE_DATASET['folder']}/WDPA_WDOECM_marine.zip", gpkg=f"{WDPA_MARINE_DATASET['folder']}/WDPA_WDOECM_marine.gpkg", + # Downloading Marine protected area database from WDPA + # extract the main zip and then merge the contained 3 zipped shapefiles + # Website: https://www.protectedplanet.net/en/thematic-areas/marine-protected-areas + message: + "Downloading Marine protected area database from WDPA" run: output_folder = Path(output["zip_file"]).parent copy2(input["zip_file"], output["zip_file"]) @@ -1172,15 +1172,15 @@ if (INSTRAT_CO2_PRICES_DATASET := dataset_version("instrat_co2_prices"))["source ]: rule retrieve_co2_prices: - message: - "Retrieving CO2 emission allowances price in EU ETS system" output: csv=f"{INSTRAT_CO2_PRICES_DATASET['folder']}/prices_eu_ets_all.csv", log: "logs/retrieve_co2_prices.log", + retries: 2 resources: mem_mb=5000, - retries: 2 + message: + "Retrieving CO2 emission allowances price in EU ETS system" run: import pandas as pd @@ -1203,12 +1203,12 @@ if ( )["source"] in ["primary", "archive"]: rule retrieve_worldbank_commodity_prices: - message: - "Retrieving monthly commodity price time series (including fossil fuels)" input: xlsx=storage(WORLD_BANK_COMMODITY_PRICES_DATASET["url"]), output: xlsx=f"{WORLD_BANK_COMMODITY_PRICES_DATASET['folder']}/CMO-Historical-Data-Monthly.xlsx", + message: + "Retrieving monthly commodity price time series (including fossil fuels)" run: copy2(input["xlsx"], output["xlsx"]) @@ -1216,8 +1216,6 @@ if ( if (TYDNP_DATASET := dataset_version("tyndp"))["source"] in ["primary", "archive"]: rule retrieve_tyndp: - message: - "Retrieving TYNDP network topology data" input: line_data=storage(TYDNP_DATASET["url"] + "/Line-data.zip"), nodes=storage(TYDNP_DATASET["url"] + "/Nodes.zip"), @@ -1228,6 +1226,8 @@ if (TYDNP_DATASET := dataset_version("tyndp"))["source"] in ["primary", "archive nodes=f"{TYDNP_DATASET['folder']}/Nodes/LIST OF NODES.xlsx", log: "logs/retrieve_tyndp.log", + message: + "Retrieving TYNDP network topology data" run: for key in input.keys(): # Keep zip file @@ -1301,8 +1301,6 @@ if OSM_DATASET["source"] in ["archive"]: OSM_ARCHIVE_FILES = get_osm_archive_files(OSM_DATASET["version"]) rule retrieve_osm_archive: - message: - "Retrieving OSM archive data" input: **{ file: storage(f"{OSM_DATASET['url']}/{file}") @@ -1315,6 +1313,8 @@ if OSM_DATASET["source"] in ["archive"]: threads: 1 resources: mem_mb=500, + message: + "Retrieving OSM archive data" run: for key in input.keys(): copy2(input[key], output[key]) @@ -1340,8 +1340,6 @@ if OSM_DATASET_INCUMBENT["source"] in ["archive"] and OSM_DATASET_INCUMBENT[ ) rule retrieve_osm_archive_incumbent: - message: - "Retrieving OSM archive incumbent data" input: **{ file: storage(f"{OSM_DATASET_INCUMBENT['url']}/{file}") @@ -1357,6 +1355,8 @@ if OSM_DATASET_INCUMBENT["source"] in ["archive"] and OSM_DATASET_INCUMBENT[ threads: 1 resources: mem_mb=500, + message: + "Retrieving OSM archive incumbent data" run: for key in input.keys(): copy2(input[key], output[key]) @@ -1373,10 +1373,6 @@ if OSM_DATASET["source"] == "build": ] rule retrieve_osm_data_raw: - message: - "Retrieving OSM electricity grid raw data for {wildcards.country}" - params: - overpass_api=config_provider("overpass_api"), output: **{ file.replace( @@ -1387,6 +1383,10 @@ if OSM_DATASET["source"] == "build": log: "logs/retrieve_osm_data_{country}.log", threads: 1 + params: + overpass_api=config_provider("overpass_api"), + message: + "Retrieving OSM electricity grid raw data for {wildcards.country}" script: scripts("retrieve_osm_data.py") @@ -1402,22 +1402,20 @@ if OSM_DATASET["source"] == "build": if (NATURA_DATASET := dataset_version("natura"))["source"] in ["archive"]: rule retrieve_natura: - message: - "Retrieving Natura 2000 raster data" input: storage(NATURA_DATASET["url"]), output: f"{NATURA_DATASET['folder']}/natura.tiff", log: "logs/retrieve_natura.log", + message: + "Retrieving Natura 2000 raster data" run: copy2(input[0], output[0]) elif NATURA_DATASET["source"] == "build": rule build_natura_raster: - message: - "Building Natura 2000 raster data" input: online=storage(NATURA_DATASET["url"]), cutout=lambda w: input_cutout(w), @@ -1425,10 +1423,12 @@ elif NATURA_DATASET["source"] == "build": zip=f"{NATURA_DATASET['folder']}/raw/natura.zip", raw=directory(f"{NATURA_DATASET['folder']}/raw"), raster=f"{NATURA_DATASET['folder']}/natura.tiff", - resources: - mem_mb=5000, log: "logs/build_natura.log", + resources: + mem_mb=5000, + message: + "Building Natura 2000 raster data" script: scripts("build_natura.py") @@ -1438,13 +1438,13 @@ if (OSM_BOUNDARIES_DATASET := dataset_version("osm_boundaries"))["source"] in [ ]: rule retrieve_osm_boundaries: - message: - "Retrieving OSM admin boundaries for {wildcards.country}" output: json=f"{OSM_BOUNDARIES_DATASET['folder']}/{country}_adm1.json", log: "logs/retrieve_osm_boundaries_{country}_adm1.log", threads: 1 + message: + "Retrieving OSM admin boundaries for {wildcards.country}" script: scripts("retrieve_osm_boundaries.py") @@ -1453,8 +1453,6 @@ elif (OSM_BOUNDARIES_DATASET := dataset_version("osm_boundaries"))["source"] in ]: rule retrieve_osm_boundaries: - message: - "Retrieving OSM admin boundaries data" input: storage( f"{OSM_BOUNDARIES_DATASET['url']}", @@ -1465,6 +1463,8 @@ elif (OSM_BOUNDARIES_DATASET := dataset_version("osm_boundaries"))["source"] in json3=f"{OSM_BOUNDARIES_DATASET['folder']}/MD_adm1.json", json4=f"{OSM_BOUNDARIES_DATASET['folder']}/BA_adm1.json", zip_file=f"{OSM_BOUNDARIES_DATASET['folder']}/osm_boundaries.zip", + message: + "Retrieving OSM admin boundaries data" run: output_folder = Path(output["zip_file"]).parent copy2(input[0], output["zip_file"]) @@ -1478,8 +1478,6 @@ if ( )["source"] in ["primary", "archive"]: rule retrieve_geothermal_heat_utilisation_potentials: - message: - "Retrieving geothermal heat utilisation potentials" input: isi_heat_potentials=storage( GEOTHERMAL_HEAT_UTILISATION_POTENTIALS_DATASET["url"] @@ -1488,8 +1486,10 @@ if ( isi_heat_potentials=f"{GEOTHERMAL_HEAT_UTILISATION_POTENTIALS_DATASET['folder']}/isi_heat_utilisation_potentials.xlsx", log: "logs/retrieve_geothermal_heat_utilisation_potentials.log", - threads: 1 retries: 2 + threads: 1 + message: + "Retrieving geothermal heat utilisation potentials" run: copy2(input["isi_heat_potentials"], output["isi_heat_potentials"]) @@ -1500,37 +1500,35 @@ if (LAU_REGIONS_DATASET := dataset_version("lau_regions"))["source"] in [ ]: rule retrieve_lau_regions: - message: - "Retrieving Local Administrative Units and Administation Unit regions" input: lau_regions=storage(LAU_REGIONS_DATASET["url"]), output: zip=f"{LAU_REGIONS_DATASET['folder']}/lau_regions.zip", log: "logs/retrieve_lau_regions.log", - threads: 1 retries: 2 + threads: 1 + message: + "Retrieving Local Administrative Units and Administation Unit regions" run: copy2(input["lau_regions"], output["zip"]) rule retrieve_seawater_temperature: - message: - "Retrieving seawater temperature data for {wildcards.year}" - params: - default_cutout=config_provider("atlite", "default_cutout"), - test_data_url=dataset_version("seawater_temperature")["url"], output: seawater_temperature="data/seawater_temperature_{year}.nc", log: "logs/retrieve_seawater_temperature_{year}.log", resources: mem_mb=10000, + params: + default_cutout=config_provider("atlite", "default_cutout"), + test_data_url=dataset_version("seawater_temperature")["url"], + message: + "Retrieving seawater temperature data for {wildcards.year}" script: scripts("retrieve_seawater_temperature.py") rule retrieve_hera_data_test_cutout: - message: - "Retrieving HERA test cutout data" input: hera_data_url=storage( f"https://zenodo.org/records/15828866/files/hera_be_2013-03-01_to_2013-03-08.zip" @@ -1538,19 +1536,19 @@ if (LAU_REGIONS_DATASET := dataset_version("lau_regions"))["source"] in [ output: river_discharge=f"data/hera_be_2013-03-01_to_2013-03-08/river_discharge_be_2013-03-01_to_2013-03-08.nc", ambient_temperature=f"data/hera_be_2013-03-01_to_2013-03-08/ambient_temp_be_2013-03-01_to_2013-03-08.nc", - params: - folder="data", log: "logs/retrieve_hera_data_test_cutout.log", + retries: 2 resources: mem_mb=10000, - retries: 2 + params: + folder="data", + message: + "Retrieving HERA test cutout data" run: unpack_archive(input[0], params.folder) rule retrieve_hera_data: - message: - "Retrieving HERA data for {wildcards.year}" input: river_discharge=storage( "https://jeodpp.jrc.ec.europa.eu/ftp/jrc-opendata/CEMS-EFAS/HERA/VER1-0/Data/NetCDF/river_discharge/dis.HERA{year}.nc" @@ -1561,13 +1559,15 @@ if (LAU_REGIONS_DATASET := dataset_version("lau_regions"))["source"] in [ output: river_discharge="data/hera_{year}/river_discharge_{year}.nc", ambient_temperature="data/hera_{year}/ambient_temp_{year}.nc", - params: - snapshot_year="{year}", log: "logs/retrieve_hera_data_{year}.log", + retries: 2 resources: mem_mb=10000, - retries: 2 + params: + snapshot_year="{year}", + message: + "Retrieving HERA data for {wildcards.year}" run: move(input.river_discharge, output.river_discharge) move(input.ambient_temperature, output.ambient_temperature) @@ -1578,8 +1578,6 @@ if (JRC_ARDECO_DATASET := dataset_version("jrc_ardeco"))["source"] in [ ]: rule retrieve_jrc_ardeco: - message: - "Retrieving JRC ARDECO data" input: ardeco_gdp=storage( f"{JRC_ARDECO_DATASET['url']}/SUVGDP?versions=2021&unit=EUR&format=csv-table" @@ -1590,6 +1588,8 @@ if (JRC_ARDECO_DATASET := dataset_version("jrc_ardeco"))["source"] in [ output: ardeco_gdp=f"{JRC_ARDECO_DATASET['folder']}/ARDECO-SUVGDP.2021.table.csv", ardeco_pop=f"{JRC_ARDECO_DATASET['folder']}/ARDECO-SNPTD.2021.table.csv", + message: + "Retrieving JRC ARDECO data" run: for key in input.keys(): copy2(input[key], output[key]) @@ -1598,8 +1598,6 @@ if (JRC_ARDECO_DATASET := dataset_version("jrc_ardeco"))["source"] in [ elif (JRC_ARDECO_DATASET := dataset_version("jrc_ardeco"))["source"] in ["archive"]: rule retrieve_jrc_ardeco: - message: - "Retrieving JRC ARDECO data" input: ardeco_gdp=storage( f"{JRC_ARDECO_DATASET['url']}/ARDECO-SUVGDP.2021.table.csv" @@ -1610,6 +1608,8 @@ elif (JRC_ARDECO_DATASET := dataset_version("jrc_ardeco"))["source"] in ["archiv output: ardeco_gdp=f"{JRC_ARDECO_DATASET['folder']}/ARDECO-SUVGDP.2021.table.csv", ardeco_pop=f"{JRC_ARDECO_DATASET['folder']}/ARDECO-SNPTD.2021.table.csv", + message: + "Retrieving JRC ARDECO data" run: for key in input.keys(): copy2(input[key], output[key]) @@ -1622,8 +1622,6 @@ if (AQUIFER_DATA_DATASET := dataset_version("aquifer_data"))["source"] in [ ]: rule retrieve_aquifer_data_bgr: - message: - "Retrieving BGR aquifer data" input: zip_file=storage(AQUIFER_DATA_DATASET["url"]), output: @@ -1640,6 +1638,8 @@ if (AQUIFER_DATA_DATASET := dataset_version("aquifer_data"))["source"] in [ "sbx", ], ), + message: + "Retrieving BGR aquifer data" run: copy2(input["zip_file"], output["zip_file"]) unpack_archive( @@ -1654,14 +1654,14 @@ if (DH_AREAS_DATASET := dataset_version("dh_areas"))["source"] in [ ]: rule retrieve_dh_areas: - message: - "Retrieving District Heating areas" input: dh_areas=storage(DH_AREAS_DATASET["url"]), output: dh_areas=f"{DH_AREAS_DATASET['folder']}/dh_areas.gpkg", log: "logs/retrieve_dh_areas.log", + message: + "Retrieving District Heating areas" run: copy2(input["dh_areas"], output["dh_areas"]) @@ -1671,21 +1671,21 @@ if (MOBILITY_PROFILES_DATASET := dataset_version("mobility_profiles"))["source"] ]: rule retrieve_mobility_profiles: - message: - "Retrieving mobility profiles data" input: kfz=storage(MOBILITY_PROFILES_DATASET["url"] + "/kfz.csv"), pkw=storage(MOBILITY_PROFILES_DATASET["url"] + "/pkw.csv"), output: kfz=f"{MOBILITY_PROFILES_DATASET['folder']}/kfz.csv", pkw=f"{MOBILITY_PROFILES_DATASET['folder']}/pkw.csv", - threads: 1 - resources: - mem_mb=1000, log: "logs/retrieve_mobility_profiles.log", benchmark: "benchmarks/retrieve_mobility_profiles" + threads: 1 + resources: + mem_mb=1000, + message: + "Retrieving mobility profiles data" run: copy2(input["kfz"], output["kfz"]) copy2(input["pkw"], output["pkw"]) diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk index 125994310..3acdfdaa8 100644 --- a/rules/solve_electricity.smk +++ b/rules/solve_electricity.smk @@ -4,15 +4,6 @@ rule solve_network: - message: - "Solving electricity network optimization for {wildcards.clusters} clusters and {wildcards.opts} electric options" - params: - solving=config_provider("solving"), - foresight=config_provider("foresight"), - co2_sequestration_potential=config_provider( - "sector", "co2_sequestration_potential", default=200 - ), - custom_extra_functionality=input_custom_extra_functionality, input: network=resources("networks/base_s_{clusters}_elec_{opts}.nc"), output: @@ -31,27 +22,26 @@ rule solve_network: python=RESULTS + "logs/solve_network/base_s_{clusters}_elec_{opts}_python.log", benchmark: (RESULTS + "benchmarks/solve_network/base_s_{clusters}_elec_{opts}") + shadow: + shadow_config threads: solver_threads resources: mem_mb=memory, runtime=config_provider("solving", "runtime", default="6h"), - shadow: - shadow_config - script: - scripts("solve_network.py") - - -rule solve_operations_network: - message: - "Solving electricity network operations optimization for {wildcards.clusters} clusters and {wildcards.opts} electric options" params: - options=config_provider("solving", "options"), solving=config_provider("solving"), foresight=config_provider("foresight"), co2_sequestration_potential=config_provider( "sector", "co2_sequestration_potential", default=200 ), custom_extra_functionality=input_custom_extra_functionality, + message: + "Solving electricity network optimization for {wildcards.clusters} clusters and {wildcards.opts} electric options" + script: + scripts("solve_network.py") + + +rule solve_operations_network: input: network=RESULTS + "networks/base_s_{clusters}_elec_{opts}.nc", output: @@ -65,11 +55,21 @@ rule solve_operations_network: + "logs/solve_operations_network/base_s_{clusters}_elec_{opts}_op_python.log", benchmark: (RESULTS + "benchmarks/solve_operations_network/base_s_{clusters}_elec_{opts}") + shadow: + shadow_config threads: 4 resources: mem_mb=memory, runtime=config_provider("solving", "runtime", default="6h"), - shadow: - shadow_config + params: + options=config_provider("solving", "options"), + solving=config_provider("solving"), + foresight=config_provider("foresight"), + co2_sequestration_potential=config_provider( + "sector", "co2_sequestration_potential", default=200 + ), + custom_extra_functionality=input_custom_extra_functionality, + message: + "Solving electricity network operations optimization for {wildcards.clusters} clusters and {wildcards.opts} electric options" script: scripts("solve_operations_network.py") diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index dcffa5c4a..1d05a841f 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -4,19 +4,6 @@ rule add_existing_baseyear: - message: - "Adding existing infrastructure for base year for {wildcards.clusters} clusters, {wildcards.planning_horizons} planning horizons, {wildcards.opts} electric options and {wildcards.sector_opts} sector options" - params: - baseyear=config_provider("scenario", "planning_horizons", 0), - sector=config_provider("sector"), - existing_capacities=config_provider("existing_capacities"), - carriers=config_provider("electricity", "renewable_carriers"), - costs=config_provider("costs"), - heat_pump_sources=config_provider("sector", "heat_pump_sources"), - energy_totals_year=config_provider("energy", "energy_totals_year"), - add_district_heating_subnodes=config_provider( - "sector", "district_heating", "subnodes", "enable" - ), input: network=lambda w: ( resources( @@ -47,6 +34,14 @@ rule add_existing_baseyear: resources( "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}_brownfield.nc" ), + log: + logs( + "add_existing_baseyear_base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.log" + ), + benchmark: + benchmarks( + "add_existing_baseyear/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}" + ) wildcard_constraints: # TODO: The first planning_horizon needs to be aligned across scenarios # snakemake does not support passing functions to wildcard_constraints @@ -55,14 +50,19 @@ rule add_existing_baseyear: threads: 1 resources: mem_mb=3000, - log: - logs( - "add_existing_baseyear_base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.log" + params: + baseyear=config_provider("scenario", "planning_horizons", 0), + sector=config_provider("sector"), + existing_capacities=config_provider("existing_capacities"), + carriers=config_provider("electricity", "renewable_carriers"), + costs=config_provider("costs"), + heat_pump_sources=config_provider("sector", "heat_pump_sources"), + energy_totals_year=config_provider("energy", "energy_totals_year"), + add_district_heating_subnodes=config_provider( + "sector", "district_heating", "subnodes", "enable" ), - benchmark: - benchmarks( - "add_existing_baseyear/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}" - ) + message: + "Adding existing infrastructure for base year for {wildcards.clusters} clusters, {wildcards.planning_horizons} planning horizons, {wildcards.opts} electric options and {wildcards.sector_opts} sector options" script: scripts("add_existing_baseyear.py") @@ -76,22 +76,6 @@ def input_profile_tech_brownfield(w): rule add_brownfield: - message: - "Adding brownfield constraints for existing infrastructure for {wildcards.clusters} clusters, {wildcards.planning_horizons} planning horizons, {wildcards.opts} electric options and {wildcards.sector_opts} sector options" - params: - H2_retrofit=config_provider("sector", "H2_retrofit"), - H2_retrofit_capacity_per_CH4=config_provider( - "sector", "H2_retrofit_capacity_per_CH4" - ), - threshold_capacity=config_provider("existing_capacities", "threshold_capacity"), - snapshots=config_provider("snapshots"), - drop_leap_day=config_provider("enable", "drop_leap_day"), - carriers=config_provider("electricity", "renewable_carriers"), - heat_pump_sources=config_provider("sector", "heat_pump_sources"), - tes=config_provider("sector", "tes"), - dynamic_ptes_capacity=config_provider( - "sector", "district_heating", "ptes", "dynamic_capacity" - ), input: unpack(input_profile_tech_brownfield), simplify_busmap=resources("busmap_base_s.csv"), @@ -110,9 +94,6 @@ rule add_brownfield: resources( "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}_brownfield.nc" ), - threads: 4 - resources: - mem_mb=10000, log: logs( "add_brownfield_base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.log" @@ -121,6 +102,25 @@ rule add_brownfield: benchmarks( "add_brownfield/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}" ) + threads: 4 + resources: + mem_mb=10000, + params: + H2_retrofit=config_provider("sector", "H2_retrofit"), + H2_retrofit_capacity_per_CH4=config_provider( + "sector", "H2_retrofit_capacity_per_CH4" + ), + threshold_capacity=config_provider("existing_capacities", "threshold_capacity"), + snapshots=config_provider("snapshots"), + drop_leap_day=config_provider("enable", "drop_leap_day"), + carriers=config_provider("electricity", "renewable_carriers"), + heat_pump_sources=config_provider("sector", "heat_pump_sources"), + tes=config_provider("sector", "tes"), + dynamic_ptes_capacity=config_provider( + "sector", "district_heating", "ptes", "dynamic_capacity" + ), + message: + "Adding brownfield constraints for existing infrastructure for {wildcards.clusters} clusters, {wildcards.planning_horizons} planning horizons, {wildcards.opts} electric options and {wildcards.sector_opts} sector options" script: scripts("add_brownfield.py") @@ -129,16 +129,6 @@ ruleorder: add_existing_baseyear > add_brownfield rule solve_sector_network_myopic: - message: - "Solving sector-coupled network with myopic foresight for {wildcards.clusters} clusters, {wildcards.planning_horizons} planning horizons, {wildcards.opts} electric options and {wildcards.sector_opts} sector options" - params: - solving=config_provider("solving"), - foresight=config_provider("foresight"), - co2_sequestration_potential=config_provider( - "sector", "co2_sequestration_potential", default=200 - ), - custom_extra_functionality=input_custom_extra_functionality, - energy_year=config_provider("energy", "energy_totals_year"), input: network=resources( "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}_final.nc" @@ -156,8 +146,6 @@ rule solve_sector_network_myopic: if config["solving"]["options"]["store_model"] else [] ), - shadow: - shadow_config log: solver=RESULTS + "logs/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}_solver.log", @@ -165,14 +153,26 @@ rule solve_sector_network_myopic: + "logs/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}_memory.log", python=RESULTS + "logs/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}_python.log", - threads: solver_threads - resources: - mem_mb=config_provider("solving", "mem_mb"), - runtime=config_provider("solving", "runtime", default="6h"), benchmark: ( RESULTS + "benchmarks/solve_sector_network/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}" ) + shadow: + shadow_config + threads: solver_threads + resources: + mem_mb=config_provider("solving", "mem_mb"), + runtime=config_provider("solving", "runtime", default="6h"), + params: + solving=config_provider("solving"), + foresight=config_provider("foresight"), + co2_sequestration_potential=config_provider( + "sector", "co2_sequestration_potential", default=200 + ), + custom_extra_functionality=input_custom_extra_functionality, + energy_year=config_provider("energy", "energy_totals_year"), + message: + "Solving sector-coupled network with myopic foresight for {wildcards.clusters} clusters, {wildcards.planning_horizons} planning horizons, {wildcards.opts} electric options and {wildcards.sector_opts} sector options" script: scripts("solve_network.py") diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk index a56dfb280..073331244 100644 --- a/rules/solve_overnight.smk +++ b/rules/solve_overnight.smk @@ -4,15 +4,6 @@ rule solve_sector_network: - message: - "Solving sector-coupled network with overnight investment optimization for {wildcards.clusters} clusters, {wildcards.planning_horizons} planning horizons, {wildcards.opts} electric options and {wildcards.sector_opts} sector options" - params: - solving=config_provider("solving"), - foresight=config_provider("foresight"), - co2_sequestration_potential=config_provider( - "sector", "co2_sequestration_potential", default=200 - ), - custom_extra_functionality=input_custom_extra_functionality, input: network=resources( "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc" @@ -28,8 +19,6 @@ rule solve_sector_network: if config["solving"]["options"]["store_model"] else [] ), - shadow: - shadow_config log: solver=RESULTS + "logs/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}_solver.log", @@ -37,14 +26,25 @@ rule solve_sector_network: + "logs/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}_memory.log", python=RESULTS + "logs/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}_python.log", - threads: solver_threads - resources: - mem_mb=config_provider("solving", "mem_mb"), - runtime=config_provider("solving", "runtime", default="6h"), benchmark: ( RESULTS + "benchmarks/solve_sector_network/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}" ) + shadow: + shadow_config + threads: solver_threads + resources: + mem_mb=config_provider("solving", "mem_mb"), + runtime=config_provider("solving", "runtime", default="6h"), + params: + solving=config_provider("solving"), + foresight=config_provider("foresight"), + co2_sequestration_potential=config_provider( + "sector", "co2_sequestration_potential", default=200 + ), + custom_extra_functionality=input_custom_extra_functionality, + message: + "Solving sector-coupled network with overnight investment optimization for {wildcards.clusters} clusters, {wildcards.planning_horizons} planning horizons, {wildcards.opts} electric options and {wildcards.sector_opts} sector options" script: scripts("solve_network.py") diff --git a/rules/solve_perfect.smk b/rules/solve_perfect.smk index 16d8203e9..afe4d241d 100644 --- a/rules/solve_perfect.smk +++ b/rules/solve_perfect.smk @@ -2,16 +2,6 @@ # # SPDX-License-Identifier: MIT rule add_existing_baseyear: - message: - "Adding existing infrastructure for base year for {wildcards.clusters} clusters, {wildcards.planning_horizons} planning horizons, {wildcards.opts} electric options and {wildcards.sector_opts} sector options" - params: - baseyear=config_provider("scenario", "planning_horizons", 0), - sector=config_provider("sector"), - existing_capacities=config_provider("existing_capacities"), - carriers=config_provider("electricity", "renewable_carriers"), - costs=config_provider("costs"), - heat_pump_sources=config_provider("sector", "heat_pump_sources"), - energy_totals_year=config_provider("energy", "energy_totals_year"), input: network=resources( "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc" @@ -34,12 +24,6 @@ rule add_existing_baseyear: resources( "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}_brownfield.nc" ), - wildcard_constraints: - planning_horizons=config["scenario"]["planning_horizons"][0], #only applies to baseyear - threads: 1 - resources: - mem_mb=config_provider("solving", "mem_mb"), - runtime=config_provider("solving", "runtime", default="24h"), log: logs( "add_existing_baseyear_base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.log" @@ -48,6 +32,22 @@ rule add_existing_baseyear: benchmarks( "add_existing_baseyear/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}" ) + wildcard_constraints: + planning_horizons=config["scenario"]["planning_horizons"][0], #only applies to baseyear + threads: 1 + resources: + mem_mb=config_provider("solving", "mem_mb"), + runtime=config_provider("solving", "runtime", default="24h"), + params: + baseyear=config_provider("scenario", "planning_horizons", 0), + sector=config_provider("sector"), + existing_capacities=config_provider("existing_capacities"), + carriers=config_provider("electricity", "renewable_carriers"), + costs=config_provider("costs"), + heat_pump_sources=config_provider("sector", "heat_pump_sources"), + energy_totals_year=config_provider("energy", "energy_totals_year"), + message: + "Adding existing infrastructure for base year for {wildcards.clusters} clusters, {wildcards.planning_horizons} planning horizons, {wildcards.opts} electric options and {wildcards.sector_opts} sector options" script: scripts("add_existing_baseyear.py") @@ -61,11 +61,6 @@ def input_network_year(w): rule prepare_perfect_foresight: - message: - "Preparing data for perfect foresight optimization for {wildcards.clusters} clusters, {wildcards.opts} electric options and {wildcards.sector_opts} sector options" - params: - costs=config_provider("costs"), - time_resolution=config_provider("clustering", "temporal", "sector"), input: unpack(input_network_year), brownfield_network=lambda w: ( @@ -78,29 +73,23 @@ rule prepare_perfect_foresight: resources( "networks/base_s_{clusters}_{opts}_{sector_opts}_brownfield_all_years.nc" ), - threads: 2 - resources: - mem_mb=10000, log: logs("prepare_perfect_foresight_{clusters}_{opts}_{sector_opts}.log"), benchmark: benchmarks("prepare_perfect_foresight_{clusters}_{opts}_{sector_opts}") + threads: 2 + resources: + mem_mb=10000, + params: + costs=config_provider("costs"), + time_resolution=config_provider("clustering", "temporal", "sector"), + message: + "Preparing data for perfect foresight optimization for {wildcards.clusters} clusters, {wildcards.opts} electric options and {wildcards.sector_opts} sector options" script: scripts("prepare_perfect_foresight.py") rule solve_sector_network_perfect: - message: - "Solving sector-coupled network with perfect foresight for {wildcards.clusters} clusters, {wildcards.opts} electric options and {wildcards.sector_opts} sector options" - params: - solving=config_provider("solving"), - foresight=config_provider("foresight"), - sector=config_provider("sector"), - planning_horizons=config_provider("scenario", "planning_horizons"), - co2_sequestration_potential=config_provider( - "sector", "co2_sequestration_potential", default=200 - ), - custom_extra_functionality=input_custom_extra_functionality, input: network=resources( "networks/base_s_{clusters}_{opts}_{sector_opts}_brownfield_all_years.nc" @@ -117,11 +106,6 @@ rule solve_sector_network_perfect: if config["solving"]["options"]["store_model"] else [] ), - threads: solver_threads - resources: - mem_mb=config_provider("solving", "mem"), - shadow: - shadow_config log: solver=RESULTS + "logs/base_s_{clusters}_{opts}_{sector_opts}_brownfield_all_years_solver.log", @@ -134,6 +118,22 @@ rule solve_sector_network_perfect: RESULTS + "benchmarks/solve_sector_network/base_s_{clusters}_{opts}_{sector_opts}_brownfield_all_years}" ) + shadow: + shadow_config + threads: solver_threads + resources: + mem_mb=config_provider("solving", "mem"), + params: + solving=config_provider("solving"), + foresight=config_provider("foresight"), + sector=config_provider("sector"), + planning_horizons=config_provider("scenario", "planning_horizons"), + co2_sequestration_potential=config_provider( + "sector", "co2_sequestration_potential", default=200 + ), + custom_extra_functionality=input_custom_extra_functionality, + message: + "Solving sector-coupled network with perfect foresight for {wildcards.clusters} clusters, {wildcards.opts} electric options and {wildcards.sector_opts} sector options" script: scripts("solve_network.py") @@ -149,8 +149,6 @@ def input_networks_make_summary_perfect(w): rule make_summary_perfect: - message: - "Creating summary for perfect foresight optimization results" input: unpack(input_networks_make_summary_perfect), costs=resources("costs_2020_processed.csv"), @@ -168,12 +166,14 @@ rule make_summary_perfect: price_statistics=RESULTS + "csvs/price_statistics.csv", metrics=RESULTS + "csvs/metrics.csv", co2_emissions=RESULTS + "csvs/co2_emissions.csv", - threads: 2 - resources: - mem_mb=10000, log: logs("make_summary_perfect.log"), benchmark: benchmarks("make_summary_perfect") + threads: 2 + resources: + mem_mb=10000, + message: + "Creating summary for perfect foresight optimization results" script: scripts("make_summary_perfect.py") diff --git a/scripts/build_biomass_potentials.py b/scripts/build_biomass_potentials.py index 796bd9e55..5736d595d 100755 --- a/scripts/build_biomass_potentials.py +++ b/scripts/build_biomass_potentials.py @@ -47,9 +47,11 @@ def _calc_unsustainable_potential(df, df_unsustainable, share_unsus, resource_ty return ( df.apply( - lambda c: c.sum() - / df.loc[df.index.str[:2] == c.name[:2]].sum().sum() - * resource_potential.loc[c.name[:2]], + lambda c: ( + c.sum() + / df.loc[df.index.str[:2] == c.name[:2]].sum().sum() + * resource_potential.loc[c.name[:2]] + ), axis=1, ) .mul(share_unsus) diff --git a/scripts/build_gas_network.py b/scripts/build_gas_network.py index 9b006ed90..285d987f0 100644 --- a/scripts/build_gas_network.py +++ b/scripts/build_gas_network.py @@ -131,8 +131,10 @@ def prepare_dataset( # lines which have way too discrepant line lengths # get assigned haversine length * length factor df["length_haversine"] = df.apply( - lambda p: length_factor - * haversine_pts([p.point0.x, p.point0.y], [p.point1.x, p.point1.y]), + lambda p: ( + length_factor + * haversine_pts([p.point0.x, p.point0.y], [p.point1.x, p.point1.y]) + ), axis=1, ) ratio = df.eval("length / length_haversine") diff --git a/scripts/build_retro_cost.py b/scripts/build_retro_cost.py index 89647e886..7b38304e4 100755 --- a/scripts/build_retro_cost.py +++ b/scripts/build_retro_cost.py @@ -496,9 +496,11 @@ def prepare_cost_retro(country_iso_dic): cost_retro[["cost_fix", "cost_var"]] = cost_retro[ ["cost_fix", "cost_var"] ].apply( - lambda x: x - * interest_rate - / (1 - (1 + interest_rate) ** -cost_retro.loc[x.index, "life_time"]) + lambda x: ( + x + * interest_rate + / (1 - (1 + interest_rate) ** -cost_retro.loc[x.index, "life_time"]) + ) ) # weightings of costs --------------------------------------------- diff --git a/scripts/build_tyndp_network.py b/scripts/build_tyndp_network.py index 1f794aef1..362df8b91 100644 --- a/scripts/build_tyndp_network.py +++ b/scripts/build_tyndp_network.py @@ -139,18 +139,18 @@ def build_shapes( # Bidding zone shapes bidding_shapes = bidding_zones.assign( bz_id=lambda df: df["zone_name"].apply(format_bz_names), - node=lambda df: df.geometry.to_crs(distance_crs) - .representative_point() - .to_crs(geo_crs), + node=lambda df: ( + df.geometry.to_crs(distance_crs).representative_point().to_crs(geo_crs) + ), x=lambda df: df["node"].x, y=lambda df: df["node"].y, ).set_index("bz_id") # Country shapes country_shapes = bidding_shapes.dissolve(by="country")[["geometry"]].assign( - node=lambda df: df.geometry.to_crs(distance_crs) - .representative_point() - .to_crs(geo_crs), + node=lambda df: ( + df.geometry.to_crs(distance_crs).representative_point().to_crs(geo_crs) + ), x=lambda df: df["node"].x, y=lambda df: df["node"].y, ) diff --git a/scripts/clean_osm_data.py b/scripts/clean_osm_data.py index fce0ce52e..a404ef0d1 100644 --- a/scripts/clean_osm_data.py +++ b/scripts/clean_osm_data.py @@ -558,7 +558,7 @@ def _import_routes_relation(path_relation): df = pd.DataFrame(data["elements"]) df["id"] = df["id"].astype(str) - df["id"] = df["id"].apply(lambda x: (f"relation/{x}")) + df["id"] = df["id"].apply(lambda x: f"relation/{x}") df["country"] = country col_tags = [ @@ -1725,10 +1725,12 @@ def _extend_lines_to_substations(gdf_lines, gdf_substations_polygon, tol=BUS_TOL gdf = ( gdf.groupby(["line_id", "voltage_line"]) .apply( - lambda x: x[["bus_id", "geometry_bus"]] - .dropna() - .set_index("bus_id")["geometry_bus"] - .to_dict(), + lambda x: ( + x[["bus_id", "geometry_bus"]] + .dropna() + .set_index("bus_id")["geometry_bus"] + .to_dict() + ), include_groups=False, ) .reset_index() diff --git a/scripts/cluster_gas_network.py b/scripts/cluster_gas_network.py index 05f527101..ebbab910d 100755 --- a/scripts/cluster_gas_network.py +++ b/scripts/cluster_gas_network.py @@ -63,8 +63,10 @@ def build_clustered_gas_network(df, bus_regions, length_factor=1.25): # recalculate lengths as center to center * length factor df["length"] = df.apply( - lambda p: length_factor - * haversine_pts([p.point0.x, p.point0.y], [p.point1.x, p.point1.y]), + lambda p: ( + length_factor + * haversine_pts([p.point0.x, p.point0.y], [p.point1.x, p.point1.y]) + ), axis=1, ) diff --git a/scripts/make_network_comparison.py b/scripts/make_network_comparison.py index 846e06fa8..fc3cda97f 100644 --- a/scripts/make_network_comparison.py +++ b/scripts/make_network_comparison.py @@ -219,9 +219,11 @@ def to_long_format(metric: str) -> pd.DataFrame: value_name="length", ) .assign( - parameter=lambda df: df["parameter"] - .str.replace(col_prefix, "", regex=False) - .map(label_map) + parameter=lambda df: ( + df["parameter"] + .str.replace(col_prefix, "", regex=False) + .map(label_map) + ) ) ) diff --git a/scripts/plot_hydrogen_network.py b/scripts/plot_hydrogen_network.py index 350631841..f1e58bdc4 100644 --- a/scripts/plot_hydrogen_network.py +++ b/scripts/plot_hydrogen_network.py @@ -36,7 +36,9 @@ def group_pipes(df, drop_direction=False): # there are pipes for each investment period rename to AC buses name for plotting df["index_orig"] = df.index df.index = df.apply( - lambda x: f"H2 pipeline {x.bus0.replace(' H2', '')} -> {x.bus1.replace(' H2', '')}", + lambda x: ( + f"H2 pipeline {x.bus0.replace(' H2', '')} -> {x.bus1.replace(' H2', '')}" + ), axis=1, ) return df.groupby(level=0).agg( @@ -108,7 +110,9 @@ def plot_h2_map(n, regions): h2_retro["index_orig"] = h2_retro.index h2_retro.index = h2_retro.apply( - lambda x: f"H2 pipeline {x.bus0.replace(' H2', '')} -> {x.bus1.replace(' H2', '')}", + lambda x: ( + f"H2 pipeline {x.bus0.replace(' H2', '')} -> {x.bus1.replace(' H2', '')}" + ), axis=1, ) diff --git a/scripts/pypsa-de/build_wasserstoff_kernnetz.py b/scripts/pypsa-de/build_wasserstoff_kernnetz.py index c4d9e811e..a59232bbd 100644 --- a/scripts/pypsa-de/build_wasserstoff_kernnetz.py +++ b/scripts/pypsa-de/build_wasserstoff_kernnetz.py @@ -366,15 +366,19 @@ def assign_locations(df, locations): # calc length of points length_factor = 1.0 df["length_haversine"] = df.apply( - lambda p: length_factor - * haversine_pts([p.point0.x, p.point0.y], [p.point1.x, p.point1.y]), + lambda p: ( + length_factor + * haversine_pts([p.point0.x, p.point0.y], [p.point1.x, p.point1.y]) + ), axis=1, ) # calc length ratio df["length_ratio"] = df.apply( - lambda row: max(row.length, row.length_haversine) - / (min(row.length, row.length_haversine) + 1), + lambda row: ( + max(row.length, row.length_haversine) + / (min(row.length, row.length_haversine) + 1) + ), axis=1, ) diff --git a/scripts/pypsa-de/cluster_wasserstoff_kernnetz.py b/scripts/pypsa-de/cluster_wasserstoff_kernnetz.py index 1b9954dbf..a6b36a80b 100644 --- a/scripts/pypsa-de/cluster_wasserstoff_kernnetz.py +++ b/scripts/pypsa-de/cluster_wasserstoff_kernnetz.py @@ -154,8 +154,10 @@ def build_clustered_h2_network( logger.info("Recalculating pipe lengths as center to center * length factor") # recalculate lengths as center to center * length factor df["length"] = df.apply( - lambda p: length_factor - * haversine_pts([p.point0.x, p.point0.y], [p.point1.x, p.point1.y]), + lambda p: ( + length_factor + * haversine_pts([p.point0.x, p.point0.y], [p.point1.x, p.point1.y]) + ), axis=1, ) @@ -182,8 +184,9 @@ def aggregate_parallel_pipes(df, aggregate_build_years="mean"): "removed_gas_cap": "sum", "ipcei": " ".join, "pci": " ".join, - "retrofitted": lambda x: (x.sum() / len(x)) - > 0.6, # consider as retrofit if more than 60% of pipes are retrofitted (relevant for costs) + "retrofitted": lambda x: ( + (x.sum() / len(x)) > 0.6 + ), # consider as retrofit if more than 60% of pipes are retrofitted (relevant for costs) } return df.groupby(df.index).agg(strategies) diff --git a/scripts/pypsa-de/plot_ariadne_report.py b/scripts/pypsa-de/plot_ariadne_report.py index bf0564350..b4b298b51 100644 --- a/scripts/pypsa-de/plot_ariadne_report.py +++ b/scripts/pypsa-de/plot_ariadne_report.py @@ -1672,7 +1672,9 @@ def group_pipes(df, drop_direction=False): # there are pipes for each investment period rename to AC buses name for plotting df["index_orig"] = df.index df.index = df.apply( - lambda x: f"H2 pipeline {x.bus0.replace(' H2', '')} -> {x.bus1.replace(' H2', '')}", + lambda x: ( + f"H2 pipeline {x.bus0.replace(' H2', '')} -> {x.bus1.replace(' H2', '')}" + ), axis=1, ) return df.groupby(level=0).agg( diff --git a/scripts/pypsa-de/plot_hydrogen_network_incl_kernnetz.py b/scripts/pypsa-de/plot_hydrogen_network_incl_kernnetz.py index 624055f9a..f286e9eb9 100644 --- a/scripts/pypsa-de/plot_hydrogen_network_incl_kernnetz.py +++ b/scripts/pypsa-de/plot_hydrogen_network_incl_kernnetz.py @@ -48,7 +48,9 @@ def group_pipes(df, drop_direction=False): # there are pipes for each investment period rename to AC buses name for plotting df["index_orig"] = df.index df.index = df.apply( - lambda x: f"H2 pipeline {x.bus0.replace(' H2', '')} -> {x.bus1.replace(' H2', '')}", + lambda x: ( + f"H2 pipeline {x.bus0.replace(' H2', '')} -> {x.bus1.replace(' H2', '')}" + ), axis=1, ) return df.groupby(level=0).agg( @@ -136,7 +138,9 @@ def plot_h2_map(n, regions): h2_retro["index_orig"] = h2_retro.index h2_retro.index = h2_retro.apply( - lambda x: f"H2 pipeline {x.bus0.replace(' H2', '')} -> {x.bus1.replace(' H2', '')}", + lambda x: ( + f"H2 pipeline {x.bus0.replace(' H2', '')} -> {x.bus1.replace(' H2', '')}" + ), axis=1, ) @@ -160,7 +164,9 @@ def plot_h2_map(n, regions): h2_kern["index_orig"] = h2_kern.index h2_kern.index = h2_kern.apply( - lambda x: f"H2 pipeline {x.bus0.replace(' H2', '')} -> {x.bus1.replace(' H2', '')}", + lambda x: ( + f"H2 pipeline {x.bus0.replace(' H2', '')} -> {x.bus1.replace(' H2', '')}" + ), axis=1, ) diff --git a/scripts/pypsa-de/prepare_district_heating_subnodes.py b/scripts/pypsa-de/prepare_district_heating_subnodes.py index a772e627f..173555574 100644 --- a/scripts/pypsa-de/prepare_district_heating_subnodes.py +++ b/scripts/pypsa-de/prepare_district_heating_subnodes.py @@ -254,9 +254,9 @@ def prepare_subnodes( axis=1, ) subnodes["lau_shape"] = subnodes.apply( - lambda x: lau.loc[ - lau.geometry.contains(x.point_coords).idxmax(), "geometry" - ].wkt, + lambda x: ( + lau.loc[lau.geometry.contains(x.point_coords).idxmax(), "geometry"].wkt + ), axis=1, ) subnodes["nuts3"] = subnodes.apply( @@ -264,9 +264,11 @@ def prepare_subnodes( axis=1, ) subnodes["nuts3_shape"] = subnodes.apply( - lambda x: heat_techs.loc[ - heat_techs.geometry.contains(x.point_coords).idxmax(), "geometry" - ].wkt, + lambda x: ( + heat_techs.loc[ + heat_techs.geometry.contains(x.point_coords).idxmax(), "geometry" + ].wkt + ), axis=1, ) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 2fc6a4a51..10e92d6c4 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -127,8 +127,10 @@ def check_p_min_p_max(p_nom_max): # adjust name to fit syntax of nominal constraint per bus df = p_nom_max.reset_index() df["name"] = df.apply( - lambda row: f"nom_max_{row['carrier']}" - + (f"_{row['build_year']}" if row["build_year"] is not None else ""), + lambda row: ( + f"nom_max_{row['carrier']}" + + (f"_{row['build_year']}" if row["build_year"] is not None else "") + ), axis=1, ) diff --git a/test/test_data_versions_layer.py b/test/test_data_versions_layer.py index 982e5c90d..86809986f 100644 --- a/test/test_data_versions_layer.py +++ b/test/test_data_versions_layer.py @@ -51,19 +51,23 @@ def sort_versions(df: pd.DataFrame) -> pd.DataFrame: error="Archive entries must have a URL", ) one_latest_per_dataset_source = Check( - lambda df: df[df["tags"].str.contains("latest")] - .groupby(["dataset", "source"]) - .size() - .eq(1) - .all(), + lambda df: ( + df[df["tags"].str.contains("latest")] + .groupby(["dataset", "source"]) + .size() + .eq(1) + .all() + ), error="Exactly one 'latest' tag required per dataset/source combination", ) latest_same_version_across_sources = Check( - lambda df: df[(df["tags"].str.contains("latest")) & (df["version"] != "unknown")] - .groupby("dataset")["version"] - .nunique() - .le(1) - .all(), + lambda df: ( + df[(df["tags"].str.contains("latest")) & (df["version"] != "unknown")] + .groupby("dataset")["version"] + .nunique() + .le(1) + .all() + ), error="All 'latest' entries for a dataset must have the same version across sources (excluding 'unknown')", ) VersionsSchema = pa.DataFrameSchema( From 9de8777799af868046658ba39003b44bebefe749 Mon Sep 17 00:00:00 2001 From: Michael Lindner Date: Wed, 8 Apr 2026 15:26:04 +0200 Subject: [PATCH 3/3] comply with snakefmt 1.0 --- Snakefile | 4 ++-- rules/pypsa-de/reporting.smk | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Snakefile b/Snakefile index cfff32a55..706a480c1 100644 --- a/Snakefile +++ b/Snakefile @@ -277,6 +277,7 @@ rule dump_graph_config: rule rulegraph: + """Generates Rule DAG in DOT, PDF, PNG, and SVG formats using the final configuration.""" input: config_file=rules.dump_graph_config.output.config_file, output: @@ -284,7 +285,6 @@ rule rulegraph: pdf=resources("dag_rulegraph.pdf"), png=resources("dag_rulegraph.png"), svg=resources("dag_rulegraph.svg"), - """Generates Rule DAG in DOT, PDF, PNG, and SVG formats using the final configuration.""" message: "Creating RULEGRAPH dag in multiple formats using the final configuration." shell: @@ -314,6 +314,7 @@ rule rulegraph: rule filegraph: + """Generates File DAG in DOT, PDF, PNG, and SVG formats using the final configuration.""" input: config_file=rules.dump_graph_config.output.config_file, output: @@ -321,7 +322,6 @@ rule filegraph: pdf=resources("dag_filegraph.pdf"), png=resources("dag_filegraph.png"), svg=resources("dag_filegraph.svg"), - """Generates File DAG in DOT, PDF, PNG, and SVG formats using the final configuration.""" message: "Creating FILEGRAPH dag in multiple formats using the final configuration." shell: diff --git a/rules/pypsa-de/reporting.smk b/rules/pypsa-de/reporting.smk index c3c546b8e..fddac0fa6 100644 --- a/rules/pypsa-de/reporting.smk +++ b/rules/pypsa-de/reporting.smk @@ -203,25 +203,25 @@ rule plot_scenario_comparison: rule compare_scenarios: input: - price_carbon="results/" - + config["run"]["prefix"] - + "/scenario_comparison/Price-Carbon.png", # expand( # RESULTS + "ariadne/capacity_detailed.png", # run=config_provider("run", "name"), # ), + price_carbon="results/" + + config["run"]["prefix"] + + "/scenario_comparison/Price-Carbon.png", rule ariadne_all: input: - expand( - RESULTS + "ariadne/report/elec_price_duration_curve.pdf", - run=config_provider("run", "name"), - ), # expand( # RESULTS + "ariadne/capacity_detailed.png", # run=config_provider("run", "name"), # ), + expand( + RESULTS + "ariadne/report/elec_price_duration_curve.pdf", + run=config_provider("run", "name"), + ), expand( RESULTS + "maps/base_s_{clusters}_{opts}_{sector_opts}-h2_network_incl_kernnetz_{planning_horizons}.pdf",