diff --git a/README.md b/README.md index f9ca564..c623486 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,17 @@ The Python package itself is described in `pyproject.toml`, which is used by Currently, LIVVext is designed to run on NERSC's Perlmutter, and ANL-LCRC's Chrysalis, but future work is planned to support other machines where E3SM runs. +## Zppy +For post-processing E3SM runs, the [zppy](https://docs.e3sm.org/zppy/_build/html/main/index.html) tool is +recommended, as it now includes a LIVVkit task, which allows for the creation of the required +climatology and timeseries files on which LIVVext depends. + +See the [zppy tutorial](https://docs.e3sm.org/zppy/_build/html/main/tutorial.html) and +[the LIVVkit example](https://github.com/E3SM-Project/zppy/blob/main/examples/post.v3.livvkit.cfg) configuration +file for further details. + +# LIVVext stand-alone workflow + ## Environment setup For setting up an environment to which LIVVext and dependencies will be @@ -35,7 +46,7 @@ environment management tool for LIVVext. First, pixi must be installed locally following [these instructions](https://pixi.prefix.dev/latest/installation/), -then an enviornment for LIVVext development can be created: +then an environment for LIVVext development can be created: ```bash $ git clone https://github.com/LIVVkit/LIVVext.git $ cd LIVVext diff --git a/config/zppy_template_r05.jinja b/config/zppy_template_r05.jinja new file mode 100644 index 0000000..f6a5da1 --- /dev/null +++ b/config/zppy_template_r05.jinja @@ -0,0 +1,744 @@ +--- +{% raw %} +common: &common + meta: &meta + Case ID: [{{ case_id }}] + Climatology years: [1980-2020] + Model: [E3SM-ELM] + climo: {{ dir_native }}/{{ case_id }}_{clim}_{sea_s}_{sea_e}_climo.nc + latlon: &climo_ann_file + {{ dir_native }}/{{ case_id }}_ANN_{sea_s}_{sea_e}_climo.nc + # These are the same for purposes of typical model run, but can be different + elevation: *climo_ann_file + lnd_climo: *climo_ann_file + topo: *climo_ann_file + glc_surf: *climo_ann_file + latv: lat + lonv: lon + topov: topo + landfracv: landfrac + img_height: 300 + {% endraw %} + # These come from zppy + year_s: {{ year1 }} + year_e: {{ year2 }} + {% raw %} + +common_smb: &common_smb + smbv: QICE + maskv: gis_mask2 + smbscale: 31536000 + cmap: BrBG + cmap_diff: BrBG + clim_even: 1 + units: mm w. e. yr^-1 + References: + - {{ livvproj_dir }}/smb/smb_icecores.bib + - {{ livvproj_dir }}/livvkit.bib + +common_energy: &common_energy + desc: Surface energy balance component {component}{comment} from {data_var_names} + references: + - {{ livvproj_dir }}/racmo/Noel2015.bib + - {{ livvproj_dir }}/cism/glissade/cism-glissade.bib + - {{ livvproj_dir }}/e3sm/Evans2019.bib + - {{ livvproj_dir }}/livvkit.bib + cmap: plasma + cmap_diff: RdBu_r + +common_racmo: &common_racmo + clim_years: + dset_a: { year_s: 1980, year_e: 2020 } + dataset_names: { model: ELM, dset_a: RACMO 2.4 } + in_dirs: { dset_a: "{{ racmo_root_dir }}/clm/{_var}" } + file_patterns: + dset_a: "{_var}_{icesheet}_{season}_{sea_s}_{sea_e}_climo.nc" + timeseries_dirs: + model: "{{ dir_ts_native }}" + dset_a: "{{ racmo_root_dir }}/ts" + ts_file_patterns: + model: "{_var}_{year_s}01_{year_e}12.nc" + dset_a: "{_var}_{icesheet}_{year_s}01_{year_e}12.nc" + masks: + dset_a: {{ livvproj_dir }}/grids/msk_{icesheet}_r025_rcm.nc + model: {{ livvproj_dir }}/grids/msk_{icesheet}_r025_rcm.nc + model_native: {{ livvproj_dir }}/grids/msk_{icesheet}_rcm_r05.nc + +common_era5: &common_era5 + climo_remap: {{ dir_era5 }}/{{ case_id }}_{clim}_{sea_s}_{sea_e}_climo.nc + dataset_names: {model_remap: 'ELM [ERA5 Grid]', model_native: ELM, dset_a: ERA5} + aavg_sort: [ELM, 'ELM [ERA5 Grid]', ERA5] + scales: {model: 1, dset_a: 1} + clim_years: + dset_a: { year_s: 1979, year_e: 2019} + in_dirs: {dset_a: {{ e3sm_diags_data_dir }}/observations/Atm/climatology/ERA5} + file_patterns: {dset_a: 'ERA5_{season}_{sea_s}_{sea_e}_climo.nc'} + masks: + dset_a: {{ livvproj_dir }}/grids/msk_{icesheet}_r025_remap_era5.traave.nc + model_remap: {{ livvproj_dir }}/grids/msk_{icesheet}_r025_remap_era5.traave.nc + model_native: {{ livvproj_dir }}/grids/msk_{icesheet}_rcm_r05.nc + +common_merra: &common_merra + climo_remap: {{ dir_merra2 }}/{{ case_id }}_{clim}_{sea_s}_{sea_e}_climo.nc + in_dirs: {dset_a: {{ e3sm_diags_data_dir }}/observations/Atm/climatology/MERRA2} + file_patterns: {dset_a: 'MERRA2_{season}_{sea_s}_{sea_e}_climo.nc'} + masks: + dset_a: {{ livvproj_dir }}/grids/msk_{icesheet}_r025_remap_merra2.traave.nc + model_remap: {{ livvproj_dir }}/grids/msk_{icesheet}_r025_remap_merra2.traave.nc + model_native: {{ livvproj_dir }}/grids/msk_{icesheet}_rcm_r05.nc + dataset_names: {model_remap: 'ELM [MERRA2 Grid]', model_native: ELM, dset_a: MERRA2} + aavg_sort: [ELM, 'ELM [MERRA2 Grid]', MERRA2] + scales: {model: 1, dset_a: 1} + clim_years: + dset_a: { year_s: 1980, year_e: 2016} + +common_ceres: &common_ceres + climo_remap: {{ dir_ceres }}/{{ case_id }}_{clim}_{sea_s}_{sea_e}_climo.nc + dataset_names: + { model_native: ELM, model_remap: "ELM [CERES Grid]", dset_a: CERES } + aavg_sort: [ELM, "ELM [CERES Grid]", CERES] + scales: { model: 1, dset_a: 1 } + clim_years: + dset_a: { year_s: 2001, year_e: 2018 } + in_dirs: + dset_a: {{ e3sm_diags_data_dir }}/observations/Atm/climatology/ceres_ebaf_surface_v4.1 + file_patterns: + dset_a: ceres_ebaf_surface_v4.1_{season}_{sea_s}_{sea_e}_climo.nc + masks: + dset_a: {{ livvproj_dir }}/grids/msk_{icesheet}_r025_remap_cmip6.traave.nc + model_remap: {{ livvproj_dir }}/grids/msk_{icesheet}_r025_remap_cmip6.traave.nc + model_native: {{ livvproj_dir }}/grids/msk_{icesheet}_rcm_r05.nc + +common_racmo_gis: &common_racmo_gis + climo_remap: {{ dir_racmo_gis }}/{{ case_id }}_{clim}_{sea_s}_{sea_e}_climo.nc + icesheet: gis + +common_racmo_ais: &common_racmo_ais + climo_remap: {{ dir_racmo_ais }}/{{ case_id }}_{clim}_{sea_s}_{sea_e}_climo.nc + icesheet: ais + +common_racmo_smb_vars: &common_racmo_smb_vars + # Surface mass balance variables for dset_a: RACMO 2.4, model: ELM + # Each field must have: + # - title: Human readable field name + # - dset_a: definition for the field in dset_a dataset (i.e. RACMO) + # - model: definition for the field in model dataset (i.e. ELM) + # - units: Units of this field + # - ac_contrib_sign: For each dataset, multiply the annual cycle by +/- 1 to achieve + # correct contribution to annual cycle of SMB + # - aavg: parameters for ice sheet-area-averaging + # - scale: Further scale the area-average, (this typically converts mm w.e./yr to GT/yr) + # - units: Units of area average (only applicable if different than units for the field) + # - sum: Perform area-weighted sum if true (area weighted mean if false) + + # Additional parameters which may be defined are: + # - mask_weight: mask variable acts as a weight (e.g. partial mask at edge of the ice sheet) + # - primary_var: True for the primary field of interest for an annual cycle plot + # - cmap: Override for colourmap for the fields + # - cmap_diff: Override for colourmap for the difference plot + # - cmin, cmax, cmin_d, cmax_d: Override for min / max values for fields and differences respectively + # - comment: added to figure caption on output (usually used to indicate signedness of fluxes) + + # Surface mass balance + - title: Surface Mass Balance + dset_a: smbgl + model: QICE + ac_contrib_sign: { model: 1, dset_a: 1 } + aavg: { scale: 1e-06, units: GT yr^-1, sum: true } + units: mm w. e. yr^-1 + mask_weight: true + primary_var: true + + # Total precipitation + - title: Total precip + dset_a: prgl + model: [+, SNOW, RAIN] + ac_contrib_sign: { model: 1, dset_a: 1 } + aavg: { scale: 1e-06, units: GT yr^-1, sum: true } + units: mm w. e. yr^-1 + cmap: YlGnBu + cmin: 0 + mask_weight: true + + # Snowfall + - title: Snowfall + dset_a: sf + model: SNOW + ac_contrib_sign: { model: 1, dset_a: 1 } + aavg: { scale: 1e-06, units: GT yr^-1, sum: true } + units: mm w. e. yr^-1 + cmap: YlGnBu + cmin: 0 + mask_weight: true + scales: + { model: 365 * 24 * 3600, dset_a: 365 * 24 * 3600, } + + # Rainfall + - title: Rain + dset_a: [+, crrate, lsrrate] + model: RAIN + ac_contrib_sign: { model: 1, dset_a: 1 } + aavg: { scale: 1e-06, units: GT yr^-1, sum: true } + units: mm w. e. yr^-1 + cmap: YlGnBu + cmin: 0 + mask_weight: true + scales: + { model: 365 * 24 * 3600, dset_a: 365 * 24 * 3600, } + + # Re-freeze + - title: Re-freeze + dset_a: rfrzgl + model: QSNOFRZ + ac_contrib_sign: { model: 1, dset_a: 1 } + aavg: { scale: 1e-06, units: GT yr^-1, sum: true } + mask_weight: true + units: mm w. e. yr^-1 + cmin: 0 + cmap: YlGnBu + + # Runoff + - title: Runoff + dset_a: totrunoff + model: QRUNOFF + ac_contrib_sign: { model: -1, dset_a: -1 } + scales: + { model: 365 * 24 * 3600, dset_a: 365 * 24 * 3600, } + aavg: { scale: 1e-06, units: GT yr^-1, sum: true } + mask_weight: true + units: mm w. e. yr^-1 + # cmin: -20.0 + cmin: 0 + # cmax: 20.0 + cmap: YlGnBu + + # Snow & ice melt + - title: Snow + ice melt + dset_a: mltgl + model: ["+", QSNOMELT, QICE_MELT] + ac_contrib_sign: { model: -1, dset_a: -1 } + scales: + { model: 365 * 24 * 3600, dset_a: 365 * 24 * 3600, } + aavg: { scale: 1e-06, units: GT yr^-1, sum: true } + mask_weight: true + units: mm w. e. yr^-1 + # cmin: -20.0 + # cmax: 20.0 + cmin: 0 + # cmax: 20.0 + cmap: YlGnBu + + # Sublimation + - title: Sublimation + dset_a: sublgl + model: QSOIL + comment: " (Positive to atmosphere)" + ac_contrib_sign: { model: -1, dset_a: 1 } + scales: { model: 365 * 24 * 3600, dset_a: -365 * 24 * 3600, } + aavg: { scale: 1e-06, units: GT yr^-1, sum: true } + mask_weight: true + units: mm w. e. yr^-1 + +common_racmo_cmb_vars: &common_racmo_cmb_vars + # Climatic mass balance variables for dset_a: RACMO 2.4, model: ELM + + # Climatic mass balance + - title: Climatic Mass Balance + dset_a: ["-", "prgl", ["-", "totrunoff", "sublgl"]] + model: ["-", ["+", "SNOW", "RAIN"], ["+", "QRUNOFF", "QSOIL"]] + ac_contrib_sign: { model: 1, dset_a: 1, } + aavg: { scale: 1e-06, units: GT yr^-1, sum: true } + units: mm w. e. yr^-1 + mask_weight: true + primary_var: true + + # Snowfall + - title: Snowfall + dset_a: sf + model: SNOW + ac_contrib_sign: { model: 1, dset_a: 1, } + aavg: { scale: 1e-06, units: GT yr^-1, sum: true } + units: mm w. e. yr^-1 + cmap: YlGnBu + cmin: 0 + mask_weight: true + scales: + { model: 365 * 24 * 3600, dset_a: 365 * 24 * 3600, } + + # Rainfall + - title: Rain + dset_a: [+, crrate, lsrrate] + model: RAIN + ac_contrib_sign: { model: 1, dset_a: 1, } + aavg: { scale: 1e-06, units: GT yr^-1, sum: true } + units: mm w. e. yr^-1 + cmap: YlGnBu + cmin: 0 + mask_weight: true + scales: + { model: 365 * 24 * 3600, dset_a: 365 * 24 * 3600, } + + # Runoff + - title: Runoff + dset_a: totrunoff + model: QRUNOFF + ac_contrib_sign: { model: -1, dset_a: -1, } + scales: + { model: 365 * 24 * 3600, dset_a: 365 * 24 * 3600, } + aavg: { scale: 1e-06, units: GT yr^-1, sum: true } + mask_weight: true + units: mm w. e. yr^-1 + cmin: 0 + cmap: YlGnBu + + # Sublimation + - title: Sublimation + dset_a: sublgl + model: QSOIL + comment: " (Positive to atmosphere)" + ac_contrib_sign: { model: -1, dset_a: 1, } + scales: + { model: 365 * 24 * 3600, dset_a: -365 * 24 * 3600, } + aavg: { scale: 1e-06, units: GT yr^-1, sum: true } + mask_weight: true + units: mm w. e. yr^-1 + +common_racmo_energy_vars: &common_racmo_energy_vars + - title: Surface temperature + dset_a: tas + model: TSA + sign: 1 + scales: {dset_a: 1, model: 1} + units: K + + - title: Albedo + dset_a: [/, rsusgl, rsds] + model: + - / + - [+, FSRND, FSRVD] + - [+, FSDSVD, FSDSND] + sign: 1 + scales: {dset_a: 1, model: 1} + units: unitless + cmin: 0.5 + cmax: 0.9 + cmin_d: -0.2 + cmax_d: 0.2 + + - title: Shortwave net + dset_a: ['-', rsds, rsusgl] + model: FSA + comment: ' (Positive to surface)' + sign: 1 + scales: {dset_a: 1, model: 1} + units: W m^-2 + + - title: Shortwave down + dset_a: rsds + model: FSDS + sign: 1 + scales: {dset_a: 1, model: 1} + units: W m^-2 + + - title: Longwave net + dset_a: strgl + model: FIRA + comment: ' (Positive to atmosphere)' + sign: 1 + scales: {dset_a: -1, model: 1} + units: W m^-2 + + - title: Longwave down + dset_a: rlds + model: FLDS + sign: 1 + scales: {dset_a: 1, model: 1} + units: W m^-2 + + - title: Sensible heat + dset_a: hfssgl + model: FSH + comment: ' (Positive to atmosphere)' + sign: 1 + scales: {dset_a: -1, model: 1} + units: W m^-2 + + - title: Latent Heat Flux + dset_a: hflsgl + model: EFLX_LH_TOT + comment: ' (Positive to atmosphere)' + sign: 1 + scales: {dset_a: -1, model: 1} + units: W m^-2 + + - title: Net energy balance + dset_a: + - + + - ['-', rsds, rsusgl] + - - + + - strgl + - [+, hfssgl, hflsgl] + model: + - '-' + - FSA + - - + + - FIRA + - [+, FSH, EFLX_LH_TOT] + sign: 1 + scales: {dset_a: 1, model: 1} + units: W m^-2 + cmin: -10 + cmax: 10 + cmap: RdBu_r + +common_era_energy_vars: &common_era_energy_vars + - title: Surface temperature + dset_a: ts + model: TSA + sign: 1 + scales: {dset_a: 1, model: 1} + units: K + + - title: Albedo + dset_a: [/, rsus, rsds] + model: + - / + - [+, FSRND, FSRVD] + - [+, FSDSVD, FSDSND] + sign: 1 + scales: {dset_a: 1, model: 1} + units: unitless + cmin: 0.5 + cmax: 1.0 + cmin_d: -0.2 + cmax_d: 0.2 + + - title: Shortwave net + dset_a: ['-', rsds, rsus] + model: FSA + comment: ' (Positive to surface)' + sign: 1 + scales: {dset_a: 1, model: 1} + units: W m^-2 + + - title: Shortwave down + dset_a: rsds + model: FSDS + sign: 1 + scales: {dset_a: 1, model: 1} + units: W m^-2 + + - model: FIRA + title: Longwave net + dset_a: ['-', rlus, rlds] + comment: ' (Positive to atmosphere)' + sign: 1 + scales: {dset_a: 1, model: 1} + units: W m^-2 + + - title: Longwave down + dset_a: rlds + model: FLDS + sign: 1 + scales: {dset_a: 1, model: 1} + units: W m^-2 + + - model: FSH + dset_a: hfss + title: Sensible heat + comment: ' (Positive to atmosphere)' + sign: 1 + scales: {dset_a: 1, model: 1} + units: W m^-2 + + - title: Latent Heat Flux + dset_a: hfls + model: EFLX_LH_TOT + comment: ' (Positive to atmosphere)' + sign: 1 + scales: {dset_a: 1, model: 1} + units: W m^-2 + + - title: Net energy balance + dset_a: + - '-' + - ['-', rsds, rsus] + - - + + - ['-', rlus, rlds] + - [+, hfss, hfls] + model: + - '-' + - FSA + - - + + - FIRA + - [+, FSH, EFLX_LH_TOT] + sign: 1 + scales: {dset_a: 1, model: 1} + units: W m^-2 + cmin: -10 + cmax: 10 + cmap: RdBu_r + +common_merra_energy_vars: &common_merra_energy_vars + - title: Surface temperature + dset_a: tas + model: TSA + sign: 1 + scales: {dset_a: 1, model: 1} + units: K + + - title: Albedo + dset_a: [/, rsus, rsds] + model: + - / + - [+, FSRND, FSRVD] + - [+, FSDSVD, FSDSND] + sign: 1 + scales: {dset_a: 1, model: 1} + units: unitless + cmin: 0.5 + cmax: 0.83 + cmin_d: -0.2 + cmax_d: 0.2 + + - title: Shortwave net + dset_a: ['-', rsds, rsus] + model: FSA + comment: ' (Positive to surface)' + sign: 1 + scales: {dset_a: 1, model: 1} + units: W m^-2 + + - title: Shortwave down + dset_a: rsds + model: FSDS + sign: 1 + scales: {dset_a: 1, model: 1} + units: W m^-2 + + - title: Longwave net + dset_a: ['-', rlus, rlds] + model: FIRA + comment: ' (Positive to atmosphere)' + sign: 1 + scales: {dset_a: 1, model: 1} + units: W m^-2 + + - title: Longwave down + dset_a: rlds + model: FLDS + sign: 1 + scales: {dset_a: 1, model: 1} + units: W m^-2 + + - title: Sensible heat + dset_a: hfss + model: FSH + comment: ' (Positive to atmosphere)' + sign: 1 + scales: {dset_a: 1, model: 1} + units: W m^-2 + + - title: Latent Heat Flux + dset_a: hfls + model: EFLX_LH_TOT + comment: ' (Positive to atmosphere)' + sign: 1 + scales: {dset_a: 1, model: 1} + units: W m^-2 + + - title: Net energy balance + dset_a: + - '-' + - ['-', rsds, rsus] + - - + + - ['-', rlus, rlds] + - [+, hfss, hfls] + model: + - '-' + - FSA + - - + + - FIRA + - [+, FSH, EFLX_LH_TOT] + sign: 1 + scales: {dset_a: 1, model: 1} + units: W m^-2 + cmin: -10 + cmax: 10 + cmap: RdBu_r + +common_ceres_energy_vars: &common_ceres_energy_vars + - title: Albedo + dset_a: [/, rsus, rsds] + model: + - / + - [+, FSRND, FSRVD] + - [+, FSDSVD, FSDSND] + sign: 1 + scales: { dset_a: 1, model: 1 } + units: unitless + cmin: 0.5 + cmax: 0.83 + cmin_d: -0.2 + cmax_d: 0.2 + + - title: Shortwave net + dset_a: sfc_net_sw_all_mon + model: FSA + comment: " (Positive to surface)" + sign: 1 + scales: { dset_a: 1, model: 1 } + units: W m^-2 + + - title: Shortwave down + dset_a: rsds + model: FSDS + sign: 1 + scales: { dset_a: 1, model: 1 } + units: W m^-2 + + - title: Longwave net + dset_a: sfc_net_lw_all_mon + model: FIRA + comment: " (Positive to atmosphere)" + sign: 1 + scales: { dset_a: -1, model: 1 } + units: W m^-2 + + - title: Longwave down + dset_a: rlds + model: FLDS + sign: 1 + scales: { dset_a: 1, model: 1 } + units: W m^-2 + +{% if gis %} +# Greenland +{% if cmb %} +Climatic_Mass_Balance_GIS: + module: {{ livvext_root }}/smb/smb_icecores.py + data_vars: *common_racmo_cmb_vars + <<: [*common, *common_smb, *common_racmo, *common_racmo_gis] + scales: + { model: 365 * 24 * 3600, dset_a: 365 * 24 * 3600 } + primary_var: Climatic Mass Balance + desc: "{component} component of CMB from {data_var_names}" +{% endif %} + +{% if smb %} +Surface_Mass_Balance_GIS: + module: {{ livvext_root }}/smb/smb_icecores.py + data_vars: *common_racmo_smb_vars + core_year_s: 1980 + core_year_e: 2021 + ib_year_e: 1987-1976 + ib_year_s: 2014-2004 + preprocess: [] + preproc_dir: {{ livvproj_dir }}/smb/processed + zwally_file: model_zwally_basins_elm_r05.csv + ib_file: IceBridge_modelXY_elm{}_r05.csv + smb_cf_file: SMB_CoreFirnEstimates_elm{}_r05.csv + smb_mo_file: SMB_Obs_Model_elm{}_r05.csv + <<: [*common, *common_smb, *common_racmo, *common_racmo_gis] + scales: + { model: 365 * 24 * 3600, dset_a: 365 * 24 * 3600 } + primary_var: smbgl + desc: "{component} component of SMB from {data_var_names}" +{% endif %} + +{% if energy_racmo %} +Energy_Balance_RACMO_GIS: + module: {{ livvext_root }}/energy/energy.py + <<: [*common, *common_energy, *common_racmo, *common_racmo_gis] + scales: {model: 1, dset_a: 1} + data_vars: *common_racmo_energy_vars +{% endif %} + +{% if energy_era5 %} +Energy_Balance_ERA5_GIS: + module: {{ livvext_root }}/energy/energy.py + icesheet: gis + mask_ocean: {model_native: false, model_remap: true, dset_a: true} + <<: [*common, *common_energy, *common_era5] + data_vars: *common_era_energy_vars +{% endif %} + +{% if energy_merra2 %} +Energy_Balance_MERRA2_GIS: + module: {{ livvext_root }}/energy/energy.py + icesheet: gis + <<: [*common, *common_energy, *common_merra] + mask_ocean: {model_native: false, model_remap: true, dset_a: true} + data_vars: *common_merra_energy_vars +{% endif %} + +{% if energy_ceres %} +Energy_Balance_CERES_GIS: + module: {{ livvext_root }}/energy/energy.py + icesheet: gis + <<: [*common, *common_energy, *common_ceres] + mask_ocean: { model_native: false, model_remap: true, dset_a: true } + data_vars: *common_ceres_energy_vars +{% endif %} +{% endif %} + +{% if ais %} +# Antarctica +{% if cmb %} +Climatic_Mass_Balance_AIS: + module: {{ livvext_root }}/smb/smb_icecores.py + data_vars: *common_racmo_cmb_vars + <<: [*common, *common_smb, *common_racmo, *common_racmo_ais] + scales: + { model: 365 * 24 * 3600, dset_a: 365 * 24 * 3600 } + primary_var: Climatic Mass Balance + desc: "{component} component of CMB from {data_var_names}" +{% endif %} + +{% if smb %} +Surface_Mass_Balance_AIS: + module: {{ livvext_root }}/smb/smb_icecores.py + data_vars: *common_racmo_smb_vars + <<: [*common, *common_smb, *common_racmo, *common_racmo_ais] + scales: + { model: 365 * 24 * 3600, dset_a: 365 * 24 * 3600 } + primary_var: smbgl + desc: "{component} component of SMB from {data_var_names}" +{% endif %} + +{% if energy_racmo %} +Energy_Balance_RACMO_AIS: + module: {{ livvext_root }}/energy/energy.py + <<: [*common, *common_energy, *common_racmo, *common_racmo_ais] + scales: {model: 1, dset_a: 1} + data_vars: *common_racmo_energy_vars +{% endif %} + +{% if energy_era5 %} +Energy_Balance_ERA5_AIS: + module: {{ livvext_root }}/energy/energy.py + icesheet: ais + mask_ocean: {model_native: false, model_remap: true, dset_a: true} + <<: [*common, *common_energy, *common_era5] + data_vars: *common_era_energy_vars +{% endif %} + +{% if energy_merra2 %} +Energy_Balance_MERRA2_AIS: + module: {{ livvext_root }}/energy/energy.py + icesheet: ais + <<: [*common, *common_energy, *common_merra] + mask_ocean: {model_native: false, model_remap: true, dset_a: true} + data_vars: *common_merra_energy_vars +{% endif %} + +{% if energy_ceres %} +Energy_Balance_CERES_AIS: + module: {{ livvext_root }}/energy/energy.py + icesheet: ais + <<: [*common, *common_energy, *common_ceres] + mask_ocean: { model_native: false, model_remap: true, dset_a: true } + data_vars: *common_ceres_energy_vars +{% endif %} +{% endif %} +{% endraw %} diff --git a/livvext/__init__.py b/livvext/__init__.py index f7686e3..3385f3e 100644 --- a/livvext/__init__.py +++ b/livvext/__init__.py @@ -31,5 +31,5 @@ Storage for global variables. These are set upon startup in the options module """ -__version_info__ = (1, 0, 2) +__version_info__ = (1, 1, 0) __version__ = ".".join(str(vi) for vi in __version_info__) diff --git a/livvext/annual_cycle.py b/livvext/annual_cycle.py index da03372..20f6eab 100644 --- a/livvext/annual_cycle.py +++ b/livvext/annual_cycle.py @@ -1,3 +1,5 @@ +"""Produce climatology figures of area averaged data by month.""" + import os from pathlib import Path @@ -19,6 +21,7 @@ def main(args, config): + """Load climatology for model and "observational" data sets, create plots.""" _files = [lxc.proc_climo_file(config, "climo_remap", mon) for mon in range(1, 13)] model_data = xr.open_mfdataset( _files, diff --git a/livvext/common.py b/livvext/common.py index 28e9e9b..13264e4 100644 --- a/livvext/common.py +++ b/livvext/common.py @@ -129,7 +129,7 @@ def proc_climo_file(config, file_tag, sea): LIVVkit /LEX configuration dict file_tag : str Configuration item which points to climatology filename to be formatted, usually - `climo` or `climo_remap` + ``climo`` or ``climo_remap`` sea : str Season identifier @@ -516,29 +516,29 @@ def area_avg( data : array_like Array of data to be averaged config : dict - LIVVkit configuration dictionary, at least contains the variable `maskv` - if `mask_var` is not set + LIVVkit configuration dictionary, at least contains the variable ``maskv`` + if ``mask_var`` is not set area_file : Path - Path to a netCDF file containing the grid cell area which matches `data` + Path to a netCDF file containing the grid cell area which matches ``data`` area_var : str Name of the netCDF variable which contains the area data mask_file : Path, optional - Path to a netCDF file containing the ice sheet mask whose shape matches `data`. - If not set, the mask is assumed to be in the `area_file` file + Path to a netCDF file containing the ice sheet mask whose shape matches ``data``. + If not set, the mask is assumed to be in the ``area_file`` file mask_var : str, optional Name of the netCDF variable which contains the ice sheet mask data, if not - set, then use `maskv` from `config` + set, then use ``maskv`` from ``config`` Returns ------- avg : float - Masked and area-weighted average of `data` - isheet_mask : array_like - Mask of ice sheet used in generating `avg` - area_maskice : array_like - Masked area used in generating `avg` - _data : array_like - Input `data` masked by `isheet_mask` + Masked and area-weighted average of ``data`` + isheet_mask : ``array_like`` + Mask of ice sheet used in generating ``avg`` + area_maskice : ``array_like`` + Masked area used in generating ``avg`` + _data : ``array_like`` + Input ``data`` masked by ``isheet_mask`` """ try: @@ -744,15 +744,15 @@ def compute_clevs( Parameters ---------- - data : dictionary + data : dict Masked array of data for which to compute contour levels bnds : tuple, optional Upper / lower percentiles to use for bounds (default: (5%, 95%)) even : bool, optional Use an even interval about 0 (default: False) keys : list, optional - List of keys within `data` for which bounds will be computed, default is all - keys in `data` + List of keys within ``data`` for which bounds will be computed, default is all + keys in ``data`` Returns ------- diff --git a/livvext/compare_gridded.py b/livvext/compare_gridded.py index e73c239..2169704 100644 --- a/livvext/compare_gridded.py +++ b/livvext/compare_gridded.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # coding: utf-8 -"""Compare three gridded datasets. Typically one "Model" and two "Observations" """ +"""Compare up to three gridded datasets. Typically one "Model" and 1 or 2 "Observations" """ import os @@ -164,6 +164,29 @@ def get_figure(n_dsets, proj=None, icesheet="gis"): def main(args, config, sea="ANN"): + """ + Generate comparison plots for a particular season. + + Parameters + ---------- + args : `argparse.Namespace` + Command line arguments + config : dict + LIVVkit configuration dictionary + sea : str, optional + "Season" identifier, either DJF, MAM, JJA, SON, ANN, or 1--12 for monthly data, + by default "ANN" + + Returns + ------- + list, dict + Returns the list of `livvkit.elements` + + Raises + ------ + NotImplementedError + _description_ + """ units = config.get("units", "UNITS UNKNOWN") icesheet = config.get("icesheet", "gis").lower() # List of fields (by their common name) which are averaged ann/seasonally/monthly diff --git a/livvext/energy/__init__.py b/livvext/energy/__init__.py index e69de29..630788b 100644 --- a/livvext/energy/__init__.py +++ b/livvext/energy/__init__.py @@ -0,0 +1 @@ +"""Module for analyzing energy balance over an ice sheet""" diff --git a/livvext/generate_cfg.py b/livvext/generate_cfg.py index 8196374..ada4783 100644 --- a/livvext/generate_cfg.py +++ b/livvext/generate_cfg.py @@ -13,6 +13,15 @@ def args(): + """ + Parse command line arguments. + + Returns + ------- + args : `argparse.Namespace` + Parsed command line arguments + + """ parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter ) @@ -76,10 +85,35 @@ def args(): help="Flag to be called from zppy, makes grid parsed", ) + parser.add_argument( + "--version", + action="version", + version=f"LIVVext v{livvext.__version__}", + help="Show LIVVext's version number and exit", + ) + return parser.parse_args() def gen_cfg(cfg_template, params, cfg_out): + """ + Generate and write LIVVext configuration file from template and parameters. + + Parameters + ---------- + cfg_template : `pathlib.Path` + Path to input Jinja2 template + params : `dict` + Parameters which are passed to the template + cfg_out : `pathlib.Path` + Path to which the completed template is written + + Returns + ------- + cfg_out : `pathlib.Path` + Path to which the completed template is written + + """ jenv = jinja2.Environment( loader=jinja2.FileSystemLoader(cfg_template.resolve().parent) ) @@ -119,21 +153,19 @@ def parse_sets(sheets, sets): def main(): + """Load machine defaults, determine analyses to be written, fill in LIVVext template.""" cl_args = args() mach = mache.discover_machine() mach_info = mache.MachineInfo() defaults = { "chrysalis": { - "livvproj_dir": Path("/lcrc/group/e3sm/livvkit"), "model_ts_dir": Path("/lcrc/group/e3sm/ac.zender/scratch/livvkit"), "grid_dir": Path("/lcrc/group/e3sm/zender/grids"), }, "pm-cpu": { - "livvproj_dir": Path("/global/cfs/cdirs/e3sm/livvkit"), "model_ts_dir": Path("/global/cfs/projectdirs/e3sm/zender/livvkit"), "grid_dir": Path("/global/cfs/cdirs/e3sm/zender/grids"), - "racmo_root_dir": Path("/global/cfs/cdirs/fanssie/racmo/2.4.1"), }, } climo_dirs = {} @@ -157,6 +189,18 @@ def main(): _mach_defaults["e3sm_diags_data_dir"] = Path( mach_info.config.get("diagnostics", "base_path") ) + _mach_defaults["livvproj_dir"] = Path( + mach_info.config.get("diagnostics", "base_path"), + "livvkit_data", + ) + _mach_defaults["racmo_root_dir"] = Path( + mach_info.config.get("diagnostics", "base_path"), + "observations", + "Land", + "racmo", + "2.4.1", + ) + params = { **_mach_defaults, "case_id": cl_args.case, diff --git a/livvext/smb/__init__.py b/livvext/smb/__init__.py index e69de29..a84ee88 100644 --- a/livvext/smb/__init__.py +++ b/livvext/smb/__init__.py @@ -0,0 +1 @@ +"""Module for analyzing the surface mass balance over an ice sheet.""" diff --git a/livvext/smb/smb_icecores.py b/livvext/smb/smb_icecores.py index e46602d..d00b571 100644 --- a/livvext/smb/smb_icecores.py +++ b/livvext/smb/smb_icecores.py @@ -37,6 +37,7 @@ from livvext import annual_cycle, compare_gridded, time_series_plot from livvext.common import SEASON_NAME from livvext.common import summarize_result as sum_res +import livvext.utils as utils with fn.TempSysPath(os.path.dirname(__file__)): import smb.plot_core_hists as c_hists @@ -45,12 +46,13 @@ import smb.plot_IB_scatter as IB_scatter import smb.plot_spatial as plt_spatial import smb.preproc as preproc - import smb.utils as utils from loguru import logger PAGE_DOCS = { - "gis": """Validation of the Greenland Ice Sheet (GrIS) surface mass balance by + # Documentation summary for SMB analysis + "smbgl": { + "gis": """Validation of the Greenland Ice Sheet (GrIS) surface mass balance by comparing modeled surface mass balance to estimates from in situ measurements and airborne radar. @@ -67,9 +69,19 @@ Some figures below are delineated by drainage basins, which are based on Zwally et al. (2012). """, - "ais": """Validation of the Antarctic Ice Sheet (AIS) surface mass balance by -comparison to RACMO reanalysis. + "ais": """Validation of the Antarctic Ice Sheet (AIS) surface mass balance by +comparison to gridded RACMO reanalysis. """, + }, + # Documentation summary for CMB analysis + "Climatic Mass Balance": { + "gis": """Validation of the Greenland Ice Sheet (GrIS) climatic mass balance by +comparison to gridded RACMO reanalysis. [CMB = (Precip - (Runoff + Sublimation)] +""", + "ais": """Validation of the Antarctic Ice Sheet (AIS) climatic mass balance by +comparison to gridded RACMO reanalysis. [CMB = (Precip - (Runoff + Sublimation)] +""", + }, } base_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..") @@ -197,7 +209,7 @@ def _format_table(x): logger.info(f"FINISHED SMB_ICECORES WITH OUTPUT TO {img_dir}") return el.Page( name, - PAGE_DOCS[config.get("icesheet", "gis")], + PAGE_DOCS[config.get("primary_var", "smbgl")][config.get("icesheet", "gis")], elements=[run_summary, el.Tabs(tabs)], ) diff --git a/livvext/utils.py b/livvext/utils.py index 88a69cc..ae5cbad 100644 --- a/livvext/utils.py +++ b/livvext/utils.py @@ -1,29 +1,31 @@ # coding=utf-8 - -from __future__ import absolute_import, print_function, unicode_literals +"""Utilities for generating LIVVkit reports of LIVVext results.""" import operator as op +from collections.abc import Iterable +from functools import singledispatch import pybtex.database import pybtex.io -import six from pybtex.backends.html import Backend as BaseBackend from pybtex.style.formatting.plain import Style as PlainStyle class HTMLBackend(BaseBackend): + """Extends ``pybtex.backends.html.Backend``""" + def __init__(self, *args, **kwargs): - super(HTMLBackend, self).__init__(*args, **kwargs) + super().__init__() self._html = "" def output(self, html): + """Append HTML to the _html attribute.""" self._html += html def format_protected(self, text): if text[:4] == "http": return self.format_href(text, text) - else: - return r'{}'.format(text) + return f'{text}' def write_prologue(self): self.output('
') @@ -40,31 +42,41 @@ def _repr_html(self, formatted_bibliography): return self._html.replace("\n", " ").replace("\\url = 3: - operator, *operands = expr + _, *operands = expr else: - operator, operands = expr + _, operands = expr operand_queue = [] for _operand in operands: @@ -227,9 +238,9 @@ def apply_operator(operands, operator, name=False): Returns ------- - output : (type(operands), str) + output : (type(``operands``), str) Returns the result of the mathematical expression, with the same type as - `operands[0]` or a string representation of the expression + ``operands[0]`` or a string representation of the expression """ ops = { @@ -262,7 +273,7 @@ def extract_name(expr): ---------- expr : list List of expressions where first element is operator, subsequent - two elements are operands, either numeric values, fields within `dset`, + two elements are operands, either numeric values, fields within ``dset``, or an expression of this kind. (e.g. ["^", ["+", ["*", "U", "U"], ["*", "V", "V"]], "0.5"] for the wind velocity) diff --git a/pyproject.toml b/pyproject.toml index 8b64a18..6cdd5c1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -152,7 +152,10 @@ html-output = "./docs" docformat = "numpy" intersphinx = [ "https://docs.python.org/3/objects.inv", - "http://xarray.pydata.org/en/latest/objects.inv" + "https://xarray.pydata.org/en/latest/objects.inv", + "https://docs.pybtex.org/objects.inv", + "https://livvkit.github.io/Docs/objects.inv", + ] theme = "readthedocs" privacy = ["PRIVATE:**.__*__", "PUBLIC:**.__init__", "PRIVATE:**.ipynb_checkpoints"] diff --git a/tests/cfg_for_tests.jinja b/tests/cfg_for_tests.jinja new file mode 100644 index 0000000..541012e --- /dev/null +++ b/tests/cfg_for_tests.jinja @@ -0,0 +1,23 @@ +common: &common + meta: &meta + Case ID: [{{ case_id }}] + Climatology years: [1980-2020] + Model: [E3SM-ELM] + climo: {{ case_out_dir }}/{{ case_id }}.{clim}_mean.nc + topo: {{ case_out_dir }}/{{ case_id }}.ANN_mean.nc + latv: lat + lonv: lon + topov: topo + +{% if run_gis %} +# Greenland +{% if set_cmb %} +Climatic_Mass_Balance_GIS: + module: livvext/smb/smb_icecores.py + <<: [*common] + scales: + { model: 365 * 24 * 3600, dset_a: 365 * 24 * 3600 } + primary_var: Climatic Mass Balance + desc: "{component} component of CMB from {data_var_names}" +{% endif %} +{% endif %} diff --git a/tests/example.bib b/tests/example.bib new file mode 100644 index 0000000..4ec4b00 --- /dev/null +++ b/tests/example.bib @@ -0,0 +1,11 @@ +@article{example, +AUTHOR = {Kelleher, M. E. and Mahajan, S.}, +TITLE = {Enhanced climate reproducibility testing with false discovery rate correction}, +JOURNAL = {Earth System Dynamics}, +VOLUME = {17}, +YEAR = {2026}, +NUMBER = {1}, +PAGES = {23--39}, +URL = {https://esd.copernicus.org/articles/17/23/2026/}, +DOI = {10.5194/esd-17-23-2026} +} diff --git a/tests/example2.bib b/tests/example2.bib new file mode 100644 index 0000000..e41a8ec --- /dev/null +++ b/tests/example2.bib @@ -0,0 +1,14 @@ +@article{MAHAJAN2017, +title = "Exploring an Ensemble-Based Approach to Atmospheric Climate Modeling and Testing at Scale", +journal = "Procedia Computer Science", +volume = "108", +pages = "735 - 744", +year = "2017", +note = "International Conference on Computational Science, ICCS 2017, 12-14 June 2017, Zurich, Switzerland", +issn = "1877-0509", +doi = "https://doi.org/10.1016/j.procs.2017.05.259", +url = "http://www.sciencedirect.com/science/article/pii/S1877050917308906", +author = "Salil Mahajan and Abigail L. Gaddis and Katherine J. Evans and Matthew R. Norman", +keywords = "reproducibility, climate simulation, ensemble testing", +abstract = "A strict throughput requirement has placed a cap on the degree to which we can depend on the execution of single, long, fine spatial grid simulations to explore global atmospheric climate behavior. Alternatively, running an ensemble of short simulations is computationally more efficient. We test the null hypothesis that the climate statistics of a full-complexity atmospheric model derived from an ensemble of independent short simulation is equivalent to that from an equilibrated long simulation. The climate of short simulation ensembles is statistically distinguishable from that of a long simulation in terms of the distribution of global annual means, largely due to the presence of low-frequency atmospheric intrinsic variability in the long simulation. We also find that model climate statistics of the simulation ensemble are sensitive to the choice of compiler optimizations. While some answer-changing optimization choices do not effect the climate state in terms of mean, variability and extremes, aggressive optimizations can result in significantly different climate states." +} diff --git a/tests/test_gen_config.py b/tests/test_gen_config.py new file mode 100644 index 0000000..adf9bff --- /dev/null +++ b/tests/test_gen_config.py @@ -0,0 +1,110 @@ +from pathlib import Path + +import livvkit.util.functions as fcn + +import livvext.generate_cfg as lxg + + +def test_parse_sets(): + set_1 = "set_racmo_ais, set_racmo_gis, set_testing1" + set_2 = "set_testing2,set_testing3" + sheet_a = "run_gis,run_ais" + sheet_b = "run_ais" + + truth_set_1 = {"set_racmo_ais": True, "set_racmo_gis": True, "set_testing1": True} + truth_set_2 = {"set_testing2": True, "set_testing3": True} + truth_sheet_a = {"run_gis": True, "run_ais": True} + truth_sheet_b = {"run_ais": True} + + param_a1 = lxg.parse_sets(sheet_a, set_1) + param_a2 = lxg.parse_sets(sheet_a, set_2) + param_b1 = lxg.parse_sets(sheet_b, set_1) + param_b2 = lxg.parse_sets(sheet_b, set_2) + + assert all([_sheet in param_a1 for _sheet in truth_sheet_a]), ( + f"MISSING ICESHEETS: {param_a1}" + ) + assert all([_sheet in param_a2 for _sheet in truth_sheet_a]), ( + f"MISSING ICESHEETS: {param_a2}" + ) + assert all([_sheet in param_b1 for _sheet in truth_sheet_b]), ( + f"MISSING ICESHEETS: {param_b1}" + ) + assert all([_sheet in param_b2 for _sheet in truth_sheet_b]), ( + f"MISSING ICESHEETS: {param_b2}" + ) + + assert all([_set in param_a1 for _set in truth_set_1]), ( + f"MISSING DATASET(S): {param_a1}" + ) + assert all([_set in param_a2 for _set in truth_set_2]), ( + f"MISSING DATASET(S): {param_a2}" + ) + assert all([_set in param_b1 for _set in truth_set_1]), ( + f"MISSING DATASET(S): {param_b1}" + ) + assert all([_set in param_b2 for _set in truth_set_2]), ( + f"MISSING DATASET(S): {param_b2}" + ) + + +def test_gen_cfg(): + expected = ( + "common: &common\n meta: &meta\n Case ID: [SimpleTest]\n " + "Climatology years: [1980-2020]\n Model: [E3SM-ELM]\n climo: " + "/data/caseout/SimpleTest/SimpleTest.{clim}_mean.nc\n topo: " + "/data/caseout/SimpleTest/SimpleTest.ANN_mean.nc\n latv: lat\n " + "lonv: lon\n topov: topo\n\n\n# Greenland\n\nClimatic_Mass_Balance_GIS:\n " + "module: livvext/smb/smb_icecores.py\n <<: [*common]\n scales:\n " + "{ model: 365 * 24 * 3600, dset_a: 365 * 24 * 3600 }\n primary_var: Climatic " + 'Mass Balance\n desc: "{component} component of CMB from {data_var_names}"\n\n' + ) + + params = { + "case_id": "SimpleTest", + "case_out_dir": "/data/caseout/SimpleTest", + "run_gis": True, + "set_cmb": True, + } + test_template = Path("tests/cfg_for_tests.jinja") + test_output_file = Path("tests/cfg_for_tests_output.yml") + + _test_output = lxg.gen_cfg(test_template, params, test_output_file) + assert _test_output == test_output_file + with open(_test_output, "r", encoding="utf-8") as _ftest: + generated = _ftest.read() + + assert generated == expected + + generated_cfg = fcn.read_yaml(_test_output) + expected_cfg = { + "common": { + "meta": { + "Case ID": ["SimpleTest"], + "Climatology years": ["1980-2020"], + "Model": ["E3SM-ELM"], + }, + "climo": "/data/caseout/SimpleTest/SimpleTest.{clim}_mean.nc", + "topo": "/data/caseout/SimpleTest/SimpleTest.ANN_mean.nc", + "latv": "lat", + "lonv": "lon", + "topov": "topo", + }, + "Climatic_Mass_Balance_GIS": { + "meta": { + "Case ID": ["SimpleTest"], + "Climatology years": ["1980-2020"], + "Model": ["E3SM-ELM"], + }, + "climo": "/data/caseout/SimpleTest/SimpleTest.{clim}_mean.nc", + "topo": "/data/caseout/SimpleTest/SimpleTest.ANN_mean.nc", + "latv": "lat", + "lonv": "lon", + "topov": "topo", + "module": "livvext/smb/smb_icecores.py", + "scales": {"model": "365 * 24 * 3600", "dset_a": "365 * 24 * 3600"}, + "primary_var": "Climatic Mass Balance", + "desc": "{component} component of CMB from {data_var_names}", + }, + } + assert expected_cfg == generated_cfg diff --git a/tests/test_utils.py b/tests/test_utils.py index eef1adb..72e0b77 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,5 +1,5 @@ import numpy as np - +import pybtex import livvext.utils as lxu ### DEFINE FORMULAS @@ -114,3 +114,46 @@ def test_vars(): assert sorted(lxu.extract_vars(seb_merra)) == sorted( ["rsds", "rsus", "rlus", "rlds", "hfss", "hfls"] ) + + +def test_bib2html(): + expected = ( + '
1
M. E. Kelleher and ' + "S. Mahajan. Enhanced climate reproducibility testing with false " + "discovery rate correction. Earth System Dynamics, 17(1):23–39, " + '2026. URL: ' + "https://esd.copernicus.org/articles/17/23/2026/, " + 'doi:10.5194/esd-17-23-2026' + ".
" + ) + expected_two = ( + '
1
M. E. Kelleher and S. ' + "Mahajan. Enhanced climate reproducibility testing with false discovery rate " + "correction. Earth System Dynamics, 17(1):23–39, 2026. URL: " + '' + "https://esd.copernicus.org/articles/17/23/2026/, " + 'doi:10.5194/esd-17-23-2026' + ".
2
Salil Mahajan, Abigail L. Gaddis, " + "Katherine J. Evans, and Matthew R. Norman. Exploring an " + "ensemble-based approach to atmospheric climate modeling and testing at scale. " + "Procedia Computer Science, 108:735 – 744, 2017. International" + " Conference on Computational Science, ICCS 2017, 12-14 June 2017, Zurich, " + "Switzerland. URL: " + '' + "http://www.sciencedirect.com/science/article/pii/S1877050917308906, " + '' + "doi:https://doi.org/10.1016/j.procs.2017.05.259.
" + ) + example_str = "tests/example.bib" + example_list = ["tests/example.bib", "tests/example2.bib"] + example_bibliography = pybtex.database.parse_file(example_str) + + assert lxu.bib2html(example_str) == expected + assert lxu.bib2html(example_list) == expected_two + assert lxu.bib2html(example_bibliography) == expected + + # Single-dispatch for is not implemented, so an exception should be raised + try: + lxu.bib2html(5) + except NotImplementedError: + pass