update and process eurostat energy balances via API (PyPSA#1987)

* Reapply "process eurostat energy balances via API"

This reverts commit 46c01e8.

* get_energy_ratio: fix query

* transformation_output_coke: handling

* new rule build_eurostat_balances (performance)

* get_energy_ratio: fix unit conversion

* fix typo

* fix typo

* pre-commit fixes

* schema fix

* add to archive

* change to build

* Revert "change to build"

This reverts commit 6dd2cdc.

---------

Co-authored-by: lkstrp <lkstrp@pm.me>

Author: fneum

data/versions.csv

@@ -46,8 +46,10 @@ eu_nuts2013,2015-12-03,primary,latest supported,2025-12-02, ,https://gisco-servi
 eu_nuts2013,2015-12-03,archive,latest supported,2026-01-13,,https://data.pypsa.org/workflows/eur/eu_nuts2013/2015-12-03/ref-nuts-2013-03m.geojson.zip
 eu_nuts2021,2021-01-01,primary,latest supported,2025-12-02,,https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/ref-nuts-2021-01m.geojson.zip
 eu_nuts2021,2021-01-01,archive,latest supported,2026-01-13,,https://data.pypsa.org/workflows/eur/eu_nuts2021/2021-01-01/ref-nuts-2021-01m.geojson.zip
-eurostat_balances,2023-04,primary,latest supported broken-link,2025-12-02,"The link is broken, use the archived versions instead.",https://ec.europa.eu/eurostat/documents/38154/4956218/Balances-April2023.zip
-eurostat_balances,2023-04,archive,latest supported,2026-01-13,,https://data.pypsa.org/workflows/eur/eurostat_balances/2023-04/balances.zip
+eurostat_balances,unknown,primary,latest supported,2026-02-04,,https://ec.europa.eu/eurostat/api/dissemination/sdmx/2.1/data/nrg_bal_c?format=TSV&compressed=true
+eurostat_balances,2026-02,archive,latest supported,2026-02-09,,https://data.pypsa.org/workflows/eur/eurostat_balances/2026-02/nrg_bal_c
+eurostat_balances,2023-04,primary,not-supported,2025-12-02,"The link is broken, use the archived versions instead.",https://ec.europa.eu/eurostat/documents/38154/4956218/Balances-April2023.zip
+eurostat_balances,2023-04,archive,not-supported,2026-01-13,,https://data.pypsa.org/workflows/eur/eurostat_balances/2023-04/balances.zip
 eurostat_household_balances,unknown,primary,latest supported,2025-12-02,"URL limits the period to 2013-2022, but the data is updated regularly.",https://ec.europa.eu/eurostat/databrowser-backend/api/extraction/1.0/LIVE/false/sdmx/csv/nrg_d_hhq__custom_11480365?startPeriod=2013&endPeriod=2022
 eurostat_household_balances,2025-07-09,archive,latest supported,2026-01-13,,https://data.pypsa.org/workflows/eur/eurostat_household_balances/2025-07-09/nrg_d_hhq.csv
 gdp_per_capita,2018-02-06,primary,latest supported,2025-12-02,"Primary link is a direct download, earlier part of zenodo bundle",https://zenodo.org/records/16556029/files/GDP_per_capita_PPP_1990_2015_v2.nc

doc/data_inventory.csv

@@ -8,7 +8,7 @@
 "nitrogen_statistics","Nitrogen Statistics and Information","Statistics and information on the worldwide supply of, demand for, and flow of the mineral commodity nitrogen.","United States Geological Survey (USGS)","https://www.usgs.gov/centers/nmic/nitrogen-statistics-and-information","Public Domain"
 "eu_nuts2013","Nomenclature of Territorial Units for Statistics (NUTS) 2013 - shapefiles","Shapefiles of EU's Nomenclature of Territorial Units for Statistics (NUTS) 2013, which is a hierarchical system for dividing up the economic territory of the European Union.","eurostat","https://ec.europa.eu/eurostat/web/nuts/overview","Reuse policy following 2011/833/EU"
 "eu_nuts2021","Nomenclature of Territorial Units for Statistics (NUTS) 2021 - shapefiles","Shapefiles of EU's Nomenclature of Territorial Units for Statistics (NUTS) 2021, which is a hierarchical system for dividing up the economic territory of the European Union.","eurostat","https://ec.europa.eu/eurostat/web/nuts/overview","Reuse policy following 2011/833/EU"
-"eurostat_balances","Energy Balances","European energy balances by country and fuel, as reported by Eurostat.","eurostat","https://ec.europa.eu/eurostat/data/database","Reuse policy following 2011/833/EU ; newer versions of the same data are available as CC-BY-4.0 through the eurostat API"
+"eurostat_balances","Energy Balances","European energy balances by country and fuel, as reported by Eurostat.","eurostat","https://ec.europa.eu/eurostat/data/database","CC-BY-4.0"
 "eurostat_household_balances","Eurostat Household Energy Balances","Disaggregated final energy consumption in household - quantities (nrg_d_hhq)","eurostat","https://ec.europa.eu/eurostat/databrowser/product/page/NRG_D_HHQ","CC-BY-4.0"
 "luisa_land_cover","The LUISA base map 2018","The LUISA Base Map 2018 is a high-resolution land use/land cover map developed and produced by the Joint Research Centre of the European Commission","European Commission Joint Research Centre","https://data.jrc.ec.europa.eu/dataset/51858b51-8f27-4006-bf82-53eba35a142c","CC-BY-4.0"
 "jrc_idees","JRC-IDEES-2021","The JRC-IDEES-2021 release contains a consistent set of disaggregated energy-economy-emissions data for each Member State of the European Union, covering all sectors of the energy system for the 2000-2021 period: industry, buildings, transport, and power generation.", "European Commission Joint Research Centre", "https://data.jrc.ec.europa.eu/dataset/82322924-506a-4c9a-8532-2bdd30d69bf5", "CC-BY-4.0"

doc/release_notes.rst

@@ -9,6 +9,11 @@ Release Notes
 Upcoming Release
 ================
 
+* Download and process Eurostat energy balances from API. This replaces the discontinued ZIP bulk download.
+
+* The function `rescale_idees_from_eurostat` was removed.
+
+* New rule `build_eurostat_balances` to build energy balances from Eurostat data. Outsourced from `build_energy_totals`
 * Updated CO2 emission allowance prices data source to Instrat API for real-time pricing data.
 * Improved OSM network building process (https://github.com/PyPSA/pypsa-eur/pull/2030): Introducing support for temporal attributes (start_date, construction tags) and pure DC buses (switching stations). The interactive network map has been completely rebuilt using PyDeck/deck.gl with GPU acceleration and includes custom JS controls, offering fuzzy search, clickable OSM references, URL-based view sharing (#theme/zoom/lat/lon), and substantially improved performance. Additionally, a generalised plot from https://www.nature.com/articles/s41597-025-04550-7 enables systematic comparison of network topology changes over time.
 

doc/sector.rst

@@ -117,6 +117,11 @@ Rule ``build_tes_capacity_profiles``
 
 .. automodule:: build_tes_capacity_profiles
 
+Rule ``build_eurostat_balances``
+==============================================================================
+
+.. automodule:: build_eurostat_balances
+
 Rule ``build_energy_totals``
 ==============================================================================
 

rules/build_sector.smk

@@ -780,6 +780,24 @@ rule build_solar_thermal_profiles:
         scripts("build_solar_thermal_profiles.py")
 
 
+rule build_eurostat_balances:
+    message:
+        "Building Eurostat energy balances"
+    input:
+        tsv_gz=rules.retrieve_eurostat_balances.output["tsv_gz"],
+    output:
+        csv=resources("eurostat_energy_balances.csv"),
+    threads: 1
+    resources:
+        mem_mb=4000,
+    log:
+        logs("build_eurostat_balances.log"),
+    benchmark:
+        benchmarks("build_eurostat_balances")
+    script:
+        "../scripts/build_eurostat_balances.py"
+
+
 rule build_energy_totals:
     message:
         "Building energy totals"
@@ -793,7 +811,7 @@ rule build_energy_totals:
         swiss_transport=f"{BFS_ROAD_VEHICLE_STOCK_DATASET['folder']}/vehicle_stock.csv",
         idees=rules.retrieve_jrc_idees.output["directory"],
         district_heat_share="data/district_heat_share.csv",
-        eurostat=rules.retrieve_eurostat_balances.output["directory"],
+        eurostat=resources("eurostat_energy_balances.csv"),
         eurostat_households=rules.retrieve_eurostat_household_balances.output["csv"],
     output:
         transformation_output_coke=resources("transformation_output_coke.csv"),
@@ -859,7 +877,7 @@ rule build_biomass_potentials:
         biomass=config_provider("biomass"),
     input:
         enspreso_biomass=rules.retrieve_enspreso_biomass.output["xlsx"],
-        eurostat=rules.retrieve_eurostat_balances.output["directory"],
+        eurostat=resources("eurostat_energy_balances.csv"),
         nuts2=rules.retrieve_eu_nuts_2013.output["shapes_level_2"],
         regions_onshore=resources("regions_onshore_base_s_{clusters}.geojson"),
         nuts3_population=ancient(rules.retrieve_nuts3_population.output["gz"]),
@@ -1049,7 +1067,7 @@ rule build_industrial_production_per_country:
     input:
         ch_industrial_production="data/ch_industrial_production_per_subsector.csv",
         ammonia_production=resources("ammonia_production.csv"),
-        eurostat=rules.retrieve_eurostat_balances.output["directory"],
+        eurostat=resources("eurostat_energy_balances.csv"),
         jrc=rules.retrieve_jrc_idees.output["directory"],
     output:
         industrial_production_per_country=resources(
@@ -1594,7 +1612,7 @@ rule prepare_sector_network:
             else []
         ),
         network=resources("networks/base_s_{clusters}_elec_{opts}.nc"),
-        eurostat=rules.retrieve_eurostat_balances.output["directory"],
+        eurostat=resources("eurostat_energy_balances.csv"),
         pop_weighted_energy_totals=resources(
             "pop_weighted_energy_totals_s_{clusters}.csv"
         ),

rules/postprocess.smk

@@ -447,7 +447,7 @@ rule plot_summary:
         costs=RESULTS + "csvs/costs.csv",
         energy=RESULTS + "csvs/energy.csv",
         balances=RESULTS + "csvs/energy_balance.csv",
-        eurostat=rules.retrieve_eurostat_balances.output["directory"],
+        eurostat=resources("eurostat_energy_balances.csv"),
         co2=rules.retrieve_ghg_emissions.output["csv"],
     output:
         costs=RESULTS + "graphs/costs.svg",

rules/retrieve.smk

@@ -31,13 +31,11 @@ if (EUROSTAT_BALANCES_DATASET := dataset_version("eurostat_balances"))["source"]
         message:
             "Retrieving Eurostat balances data"
         input:
-            zip_file=storage(EUROSTAT_BALANCES_DATASET["url"]),
+            tsv_gz=storage(EUROSTAT_BALANCES_DATASET["url"]),
         output:
-            zip_file=f"{EUROSTAT_BALANCES_DATASET['folder']}/balances.zip",
-            directory=directory(EUROSTAT_BALANCES_DATASET["folder"]),
+            tsv_gz=f"{EUROSTAT_BALANCES_DATASET['folder']}/estat_nrg_bal_c.tsv.gz",
         run:
-            copy2(input["zip_file"], output["zip_file"])
-            unpack_archive(output["zip_file"], output["directory"])
+            copy2(input["tsv_gz"], output["tsv_gz"])
 
 
 if (

scripts/build_biomass_potentials.py

@@ -13,7 +13,6 @@
 import pandas as pd
 
 from scripts._helpers import configure_logging, set_scenario_config
-from scripts.build_energy_totals import build_eurostat
 
 logger = logging.getLogger(__name__)
 AVAILABLE_BIOMASS_YEARS = [2010, 2020, 2030, 2040, 2050]
@@ -246,7 +245,7 @@ def convert_nuts2_to_regions(bio_nuts2, regions):
     return bio_regions
 
 
-def add_unsustainable_potentials(df):
+def add_unsustainable_potentials(df, input_eurostat):
     """
     Add unsustainable biomass potentials to the given dataframe. The difference
     between the data of JRC and Eurostat is assumed to be unsustainable
@@ -269,36 +268,29 @@ def add_unsustainable_potentials(df):
     else:
         latest_year = 2021
     idees_rename = {"GR": "EL", "GB": "UK"}
+    year = max(min(latest_year, int(snakemake.wildcards.planning_horizons)), 1990)  # noqa: F841
     df_unsustainable = (
-        build_eurostat(
-            countries=snakemake.config["countries"],
-            input_eurostat=snakemake.input.eurostat,
-            nprocesses=int(snakemake.threads),
-        )
-        .xs(
-            max(min(latest_year, int(snakemake.wildcards.planning_horizons)), 1990),
-            level=1,
-        )
-        .xs("Primary production", level=2)
-        .droplevel([1, 2, 3])
+        pd.read_csv(input_eurostat)
+        .query("year == @year and nrg_bal == 'PPRD'")  # Primary production
+        .set_index(["country", "siec"])
+        .value.unstack("siec")
     )
 
-    df_unsustainable.index = df_unsustainable.index.str.strip()
     df_unsustainable = df_unsustainable.rename(
         {v: k for k, v in idees_rename.items()}, axis=0
     )
 
     bio_carriers = [
-        "Primary solid biofuels",
-        "Biogases",
-        "Renewable municipal waste",
-        "Pure biogasoline",
-        "Blended biogasoline",
-        "Pure biodiesels",
-        "Blended biodiesels",
-        "Pure bio jet kerosene",
-        "Blended bio jet kerosene",
-        "Other liquid biofuels",
+        "R5110-5150_W6000RI",  # Primary solid biofuels
+        "R5300",  # Biogases
+        "W6210",  # Renewable municipal waste
+        "R5210P",  # Pure biogasoline
+        "R5210B",  # Blended biogasoline
+        "R5220P",  # Pure biodiesels
+        "R5220B",  # Blended biodiesels
+        "R5230P",  # Pure bio jet kerosene
+        "R5230B",  # Blended bio jet kerosene
+        "R5290",  # Other liquid biofuels
     ]
 
     df_unsustainable = df_unsustainable[bio_carriers]
@@ -310,20 +302,26 @@ def add_unsustainable_potentials(df):
 
     # Calculate unsustainable solid biomass
     df_wo_ch["unsustainable solid biomass"] = _calc_unsustainable_potential(
-        df_wo_ch, df_unsustainable, share_unsus, "Primary solid biofuels"
+        df_wo_ch,
+        df_unsustainable,
+        share_unsus,
+        "R5110-5150_W6000RI",  # Primary solid biofuels
     )
 
     # Calculate unsustainable biogas
     df_wo_ch["unsustainable biogas"] = _calc_unsustainable_potential(
-        df_wo_ch, df_unsustainable, share_unsus, "Biogases"
+        df_wo_ch,
+        df_unsustainable,
+        share_unsus,
+        "R5300",  # Biogases
     )
 
     # Calculate unsustainable bioliquids
     df_wo_ch["unsustainable bioliquids"] = _calc_unsustainable_potential(
         df_wo_ch,
         df_unsustainable,
         share_unsus,
-        resource_type="gasoline|diesel|kerosene|liquid",
+        resource_type="R5210|R5220|R5230|R5290",  # gasoline, diesel, kerosene, liquids
     )
 
     share_sus = params.get("share_sustainable_potential_available").get(investment_year)
@@ -391,7 +389,8 @@ def add_unsustainable_potentials(df):
     grouper = {v: k for k, vv in params["classes"].items() for v in vv}
     df = df.T.groupby(grouper).sum().T
 
-    df = add_unsustainable_potentials(df)
+    input_eurostat = snakemake.input.eurostat
+    df = add_unsustainable_potentials(df, input_eurostat)
 
     df *= 1e6  # TWh/a to MWh/a
     df.index.name = "MWh/a"