From 6037d0fbcb5f71bdb21edd471540830fab34af06 Mon Sep 17 00:00:00 2001 From: Peter J Gleckler Date: Tue, 8 Apr 2025 13:58:58 -0700 Subject: [PATCH 1/6] first test --- DRCDP_institution_id.json | 4 +++ DRCDP_source_id.json | 40 +++++++++++++++++++++++++ Tables/DRCDP_CV.json | 44 +++++++++++++++++++++++++++ src/createCVs.ipynb | 31 ++++++++++++++++++-- src/createCVs.py | 62 +++++++++++++++++++++++++++++++++++++++ src/pullTable.ipynb | 4 +-- 6 files changed, 180 insertions(+), 5 deletions(-) create mode 100644 src/createCVs.py diff --git a/DRCDP_institution_id.json b/DRCDP_institution_id.json index 363e810..728a0db 100644 --- a/DRCDP_institution_id.json +++ b/DRCDP_institution_id.json @@ -1,5 +1,9 @@ { "institution_id":{ + "NCAR":{ + "contact":"Samantha Hartke; hartke@ucar.edu", + "name":"National Center for Atmospheric Research, Boulder, CO, USA" + }, "TTU":{ "ROR":"0405mnx93", "URL":"https://www.ttu.edu/", diff --git a/DRCDP_source_id.json b/DRCDP_source_id.json index 6ea377a..1294e4f 100644 --- a/DRCDP_source_id.json +++ b/DRCDP_source_id.json @@ -1,5 +1,45 @@ { "source_id":{ + "GARD-LENS-1-0":{ + "calendar":"standard", + "contact":"Samantha Hartke; hartke@ucar.edu", + "further_info_url":"https://opensky.ucar.edu/islandora/object/articles%3A42362", + "grid":"5 x 5 km latitude x longitude", + "grid_label":"gn", + "institution_id":"NCAR", + "license":"Creative Commons Attribution 4.0 International", + "license_id":"CC BY 4.0", + "license_url":"https://creativecommons.org/licenses/by/4.0/", + "nominal_resolution":"5 km", + "product":"downscaled-statistical", + "reference":"Hartke, S.H., Newman, A.J., Gutmann, E. et al. GARD-LENS: A downscaled large ensemble dataset for understanding future climate and its uncertainties. Sci Data 11, 1374 (2024). https://doi.org/10.1038/s41597-024-04205-z", + "region":"north_america", + "region_id":"NAM", + "source":"GARD-LENS-1-0: Statistically-downscaled climate model projections based on CMIP6", + "source_name":"GARD-LENS", + "source_version":"1.0", + "title":"GARD-LENS-1-0 dataset prepared for DRCDP" + }, + "GARD-LENS1-0":{ + "calendar":"standard", + "contact":"Samantha Hartke; hartke@ucar.edu", + "further_info_url":"https://opensky.ucar.edu/islandora/object/articles%3A42362", + "grid":"5 x 5 km latitude x longitude", + "grid_label":"gn", + "institution_id":"NCAR", + "license":"Creative Commons Attribution 4.0 International", + "license_id":"CC BY 4.0", + "license_url":"https://creativecommons.org/licenses/by/4.0/", + "nominal_resolution":"5 km", + "product":"downscaled-statistical", + "reference":"Hartke, S.H., Newman, A.J., Gutmann, E. et al. GARD-LENS: A downscaled large ensemble dataset for understanding future climate and its uncertainties. Sci Data 11, 1374 (2024). https://doi.org/10.1038/s41597-024-04205-z", + "region":"north_america", + "region_id":"NAM", + "source":"GARD-LENS-1-0: Statistically-downscaled climate model projections based on CMIP6", + "source_name":"GARD-LENS", + "source_version":"1.0", + "title":"GARD-LENS-1-0 dataset prepared for DRCDP" + }, "LOCA2-0":{ "calendar":"gregorian", "contact":"Daniel Feldman; DRFeldman@lbl.gov", diff --git a/Tables/DRCDP_CV.json b/Tables/DRCDP_CV.json index ae8667c..e80d07c 100644 --- a/Tables/DRCDP_CV.json +++ b/Tables/DRCDP_CV.json @@ -14,6 +14,10 @@ "gn":"data reported on the analysis-native grid" }, "institution_id":{ + "NCAR":{ + "contact":"Samantha Hartke; hartke@ucar.edu", + "name":"National Center for Atmospheric Research, Boulder, CO, USA" + }, "TTU":{ "ROR":"0405mnx93", "URL":"https://www.ttu.edu/", @@ -102,6 +106,46 @@ "variable_id" ], "source_id":{ + "GARD-LENS-1-0":{ + "calendar":"standard", + "contact":"Samantha Hartke; hartke@ucar.edu", + "further_info_url":"https://opensky.ucar.edu/islandora/object/articles%3A42362", + "grid":"5 x 5 km latitude x longitude", + "grid_label":"gn", + "institution_id":"NCAR", + "license":"Creative Commons Attribution 4.0 International", + "license_id":"CC BY 4.0", + "license_url":"https://creativecommons.org/licenses/by/4.0/", + "nominal_resolution":"5 km", + "product":"downscaled-statistical", + "reference":"Hartke, S.H., Newman, A.J., Gutmann, E. et al. GARD-LENS: A downscaled large ensemble dataset for understanding future climate and its uncertainties. Sci Data 11, 1374 (2024). https://doi.org/10.1038/s41597-024-04205-z", + "region":"north_america", + "region_id":"NAM", + "source":"GARD-LENS-1-0: Statistically-downscaled climate model projections based on CMIP6", + "source_name":"GARD-LENS", + "source_version":"1.0", + "title":"GARD-LENS-1-0 dataset prepared for DRCDP" + }, + "GARD-LENS1-0":{ + "calendar":"standard", + "contact":"Samantha Hartke; hartke@ucar.edu", + "further_info_url":"https://opensky.ucar.edu/islandora/object/articles%3A42362", + "grid":"5 x 5 km latitude x longitude", + "grid_label":"gn", + "institution_id":"NCAR", + "license":"Creative Commons Attribution 4.0 International", + "license_id":"CC BY 4.0", + "license_url":"https://creativecommons.org/licenses/by/4.0/", + "nominal_resolution":"5 km", + "product":"downscaled-statistical", + "reference":"Hartke, S.H., Newman, A.J., Gutmann, E. et al. GARD-LENS: A downscaled large ensemble dataset for understanding future climate and its uncertainties. Sci Data 11, 1374 (2024). https://doi.org/10.1038/s41597-024-04205-z", + "region":"north_america", + "region_id":"NAM", + "source":"GARD-LENS-1-0: Statistically-downscaled climate model projections based on CMIP6", + "source_name":"GARD-LENS", + "source_version":"1.0", + "title":"GARD-LENS-1-0 dataset prepared for DRCDP" + }, "LOCA2-0":{ "calendar":"gregorian", "contact":"Daniel Feldman; DRFeldman@lbl.gov", diff --git a/src/createCVs.ipynb b/src/createCVs.ipynb index c54b3c4..b7b6705 100644 --- a/src/createCVs.ipynb +++ b/src/createCVs.ipynb @@ -349,7 +349,11 @@ "] = \"Texas Tech University, 2520 Broadway Avenue, Lubbock, TX 79409, USA\"\n", "institution_id[\"institution_id\"][\"TTU\"][\"ROR\"] = \"0405mnx93\"\n", "institution_id[\"institution_id\"][\"TTU\"][\"URL\"] = \"https://www.ttu.edu/\"\n", - "institution_id" + "institution_id\n", + "\n", + "institution_id[\"institution_id\"][\"NCAR\"] = {}\n", + "institution_id[\"institution_id\"][\"NCAR\"][\"name\"] = \"National Center for Atmospheric Research, Boulder, CO, USA\"\n", + "institution_id[\"institution_id\"][\"NCAR\"][\"\"] = \"hartke@ucar.edu\"" ] }, { @@ -877,6 +881,27 @@ "source_id[\"source_id\"][\"STAR-ESDM1-0\"][\n", " \"title\"\n", "] = \"STAR-ESDM 1.0 dataset prepared for DRCDP\"\n", + "\n", + "source_id[\"source_id\"][\"source_id[\"source_id\"][\"STAR-ESDM1-0\"][\"region\"] = \"north_america\"\n", + "source_id[\"source_id\"][\"STAR-ESDM1-0\"][\"region_id\"] = \"NAM\"\n", + "source_id[\"source_id\"][\"STAR-ESDM1-0\"][\n", + " \"source\"\n", + "] = \"STAR-ESDM 1.0: Statistically-downscaled climate model projections based on CMIP6\"\n", + "source_id[\"source_id\"][\"STAR-ESDM1-0\"][\"source_name\"] = \"STAR-ESDM\"\n", + "source_id[\"source_id\"][\"STAR-ESDM1-0\"][\"source_version\"] = \"1.0\"\n", + "source_id[\"source_id\"][\"STAR-ESDM1-0\"][\n", + " \"title\"\n", + "] = \"STAR-ESDM 1.0 dataset prepared for DRCDP1-0\"][\"region\"] = \"north_america\"\n", + "source_id[\"source_id\"][\"STAR-ESDM1-0\"][\"region_id\"] = \"NAM\"\n", + "source_id[\"source_id\"][\"STAR-ESDM1-0\"][\n", + " \"source\"\n", + "] = \"STAR-ESDM 1.0: Statistically-downscaled climate model projections based on CMIP6\"\n", + "source_id[\"source_id\"][\"STAR-ESDM1-0\"][\"source_name\"] = \"STAR-ESDM\"\n", + "source_id[\"source_id\"][\"STAR-ESDM1-0\"][\"source_version\"] = \"1.0\"\n", + "source_id[\"source_id\"][\"STAR-ESDM1-0\"][\n", + " \"title\"\n", + "] = \"STAR-ESDM 1.0 dataset prepared for DRCDP\"\n", + "\n", "source_id" ] }, @@ -1056,7 +1081,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "NERSC Python", "language": "python", "name": "python3" }, @@ -1070,7 +1095,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.13.2" + "version": "3.11.7" } }, "nbformat": 4, diff --git a/src/createCVs.py b/src/createCVs.py new file mode 100644 index 0000000..20f034c --- /dev/null +++ b/src/createCVs.py @@ -0,0 +1,62 @@ +import json + + +# read data from local (ensuring a pull has occurred to sync with main) +f = open('../DRCDP_institution_id.json','r') +institution_id = json.load(f) + +with open('../DRCDP_institution_id.json','r') as f: institution_id = json.load(f) +with open('../DRCDP_source_id.json','r') as f: source_id = json.load(f) + +# add new institution_id +key = 'NCAR' +institution_id['institution_id'][key] = {} +institution_id['institution_id'][key][ 'contact' ] = 'Samantha Hartke; hartke@ucar.edu' +institution_id['institution_id'][key][ 'name' ] = 'National Center for Atmospheric Research, Boulder, CO, USA' +#institution_id['institution_id'][key]['ROR'] = '00d9ah105' +#institution_id['institution_id'][key]['URL'] = 'https://www.climatologylab.org' + +# add new source_id +key = 'GARD-LENS-1-0' +source_id['source_id'][key] = {} +source_id['source_id'][key]['calendar'] = 'standard' +source_id['source_id'][key]['contact'] = 'Samantha Hartke; hartke@ucar.edu' +source_id['source_id'][key]['further_info_url'] = 'https://opensky.ucar.edu/islandora/object/articles%3A42362' +source_id['source_id'][key]['grid'] = '5 x 5 km latitude x longitude' +source_id['source_id'][key]['grid_label'] = 'gn' +source_id['source_id'][key]['institution_id'] = 'NCAR' +source_id['source_id'][key][ 'license' ] = 'Creative Commons Attribution 4.0 International' +source_id['source_id'][key]['license_id'] = 'CC BY 4.0' +source_id['source_id'][key][ 'license_url' ] = 'https://creativecommons.org/licenses/by/4.0/' +source_id['source_id'][key]['nominal_resolution'] = '5 km' +source_id['source_id'][key]['product'] = 'downscaled-statistical' +source_id['source_id'][key]['reference'] = ' '.join( + ['Hartke, S.H., Newman, A.J., Gutmann, E. et al. GARD-LENS: A downscaled large ensemble dataset for understanding future climate and its uncertainties. Sci Data 11, 1374 (2024). https://doi.org/10.1038/s41597-024-04205-z' ] +) +source_id['source_id'][key]['region'] = 'north_america' +source_id['source_id'][key]['region_id'] = 'NAM' +source_id['source_id'][key][ 'source' ] = 'GARD-LENS-1-0: Statistically-downscaled climate model projections based on CMIP6' +source_id['source_id'][key]['source_name'] = 'GARD-LENS' +source_id['source_id'][key]['source_version'] = '1.0' +source_id['source_id'][key]['title'] = 'GARD-LENS-1-0 dataset prepared for DRCDP' + +# overwrite existing files +# write files +outFile = ''.join(['../DRCDP_institution_id.json']) +with open(outFile, 'w') as f: + json.dump(institution_id, f, ensure_ascii=True, sort_keys=True, indent=4, separators=(',', ':')) +outFile = ''.join(['../DRCDP_source_id.json']) +with open(outFile, 'w') as f: + json.dump(source_id, f, ensure_ascii=True, sort_keys=True, indent=4, separators=(',', ':')) + +# overwrite DRCDP_CV.json with new entries +f = open('../Tables/DRCDP_CV.json','r') +CV = json.load(f) +CV['CV']['institution_id'] = institution_id['institution_id'] +CV['CV']['source_id'] = source_id['source_id'] + +# overwrite DRCDP_CV.json +outFile = ''.join(['../Tables/DRCDP_CV.json']) +with open(outFile, 'w') as f: + json.dump(CV, f, ensure_ascii=True, sort_keys=True, indent=4, separators=(',', ':')) + diff --git a/src/pullTable.ipynb b/src/pullTable.ipynb index 924fd4f..75578a0 100644 --- a/src/pullTable.ipynb +++ b/src/pullTable.ipynb @@ -383,7 +383,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "NERSC Python", "language": "python", "name": "python3" }, @@ -397,7 +397,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.13.2" + "version": "3.11.7" } }, "nbformat": 4, From 1320f10886c227aecb1106f60cb035a20f88590e Mon Sep 17 00:00:00 2001 From: Peter J Gleckler Date: Tue, 8 Apr 2025 14:02:11 -0700 Subject: [PATCH 2/6] test1 --- .../GARD-LENS/GARDLENS_CMIP6_input.json | 14 +++ .../GARD-LENS/GARDLENS_CMIP6_runCMOR.py | 88 +++++++++++++++++++ DataPreparationExamples/GARD-LENS/jobs.txt | 22 +++++ 3 files changed, 124 insertions(+) create mode 100644 DataPreparationExamples/GARD-LENS/GARDLENS_CMIP6_input.json create mode 100644 DataPreparationExamples/GARD-LENS/GARDLENS_CMIP6_runCMOR.py create mode 100755 DataPreparationExamples/GARD-LENS/jobs.txt diff --git a/DataPreparationExamples/GARD-LENS/GARDLENS_CMIP6_input.json b/DataPreparationExamples/GARD-LENS/GARDLENS_CMIP6_input.json new file mode 100644 index 0000000..c9ad7a6 --- /dev/null +++ b/DataPreparationExamples/GARD-LENS/GARDLENS_CMIP6_input.json @@ -0,0 +1,14 @@ +{ + "_AXIS_ENTRY_FILE":"DRCDP_coordinate.json", + "_FORMULA_VAR_FILE":"DRCDP_formula_terms.json", + "_controlled_vocabulary_file":"DRCDP_CV.json", + "activity_id": "DRCDP", + "source_id": "GARD-LENS-1-0", + "institution_id":"NCAR", + "driving_activity_id": "CMIP", + "driving_experiment_id": "historical", + "driving_mip_era": "CMIP6", + "driving_source_id": "ACCESS-CM2", + "driving_variant_label": "r1i1p1f1", + "outpath":"/global/cfs/projectdirs/m3522/project_downscale/" +} diff --git a/DataPreparationExamples/GARD-LENS/GARDLENS_CMIP6_runCMOR.py b/DataPreparationExamples/GARD-LENS/GARDLENS_CMIP6_runCMOR.py new file mode 100644 index 0000000..832ef0b --- /dev/null +++ b/DataPreparationExamples/GARD-LENS/GARDLENS_CMIP6_runCMOR.py @@ -0,0 +1,88 @@ +import cmor +import xcdat as xc +import xarray as xr +import cftime +import numpy as np +import sys, os, glob +from datetime import datetime + + +multi = False +if multi == True: + vari = sys.argv[1] + domain = sys.argv[2] + modi = sys.argv[3] + rn = sys.argv[4] + + inputVarName = vari + outputVarName = vari + if vari == 'pr': outputUnits = 'kg m-2 s-1' + if vari == 'tasmax': outputUnits = 'K' + if vari == 'tasmin': outputUnits = 'K' + + +vari = 'pcp' +domain = 'CONUS' +mod = 'ecearth3' +rn = 'r150i1p1f1' + +inputVarName = vari +outputVarName = 'pr' +outputUnits = 'kg m-2 s-1' + + +cmorTable = '../../Tables/DRCDP_APday.json' +inputJson = 'GARDLENS_CMIP6_input.json' + +inFile = '/global/cfs/projectdirs/m2637/GARDLENS/' + vari + '/GARDLENS_' + mod + '_' + rn + '_' + vari + '_1970_2100_' + domain + '.nc' + +start_time = datetime.now() +#fc = xc.open_mfdataset(inFile,decode_times=True,use_cftime=True, preprocess=extract_date) +fc = xc.open_dataset(inFile,decode_times=False,use_cftime=False) + +f = fc +f = fc.isel(time=slice(0,100)) +d = f[inputVarName] + +lat = f.lat.values +lon = f.lon.values +time = f.time.values +tunits = "days since 1900-01-01" + +f = f.bounds.add_bounds("X") +f = f.bounds.add_bounds("Y") +f = f.bounds.add_bounds("T") + +##### CMOR setup +cmor.setup(inpath='./',netcdf_file_action=cmor.CMOR_REPLACE_4,logfile= vari + '_' + mod + '-' + rn + '-'+'cmorLog.txt') +cmor.dataset_json(inputJson) +cmor.load_table(cmorTable) + +# SET CMIP MODEL SPECIFIC ATTRIBUTES +#cmor.set_cur_dataset_attribute("source_id","LOCA2--" + mod) +cmor.set_cur_dataset_attribute("driving_source_id",mod) +cmor.set_cur_dataset_attribute("driving_variant_label",rn) +cmor.set_cur_dataset_attribute("driving_experiment_id",'historical') + +# Create CMOR axes +cmorLat = cmor.axis("latitude", coord_vals=lat[:], cell_bounds=f.lat_bnds.values, units="degrees_north") +cmorLon = cmor.axis("longitude", coord_vals=lon[:], cell_bounds=f.lon_bnds.values, units="degrees_east") +#cmorTime = cmor.axis("time", coord_vals=cftime.date2num(time,tunits), cell_bounds=cftime.date2num(f.time_bnds.values,tunits), units= tunits) +cmorTime = cmor.axis("time", coord_vals=time[:], cell_bounds=f.time_bnds.values[:], units= tunits) +cmoraxes = [cmorTime,cmorLat, cmorLon] + +# Setup units and create variable to write using cmor - see https://cmor.llnl.gov/mydoc_cmor3_api/#cmor_set_variable_attribute +varid = cmor.variable(outputVarName,outputUnits,cmoraxes,missing_value=1.e20) +values = np.array(d[:],np.float32) + +cmor.set_variable_attribute(varid,'valid_min','f',2.0) +cmor.set_variable_attribute(varid,'valid_max','f',3.0) + +cmor.set_deflate(varid,1,1,1) ; # shuffle=1,deflate=1,deflate_level=1 - Deflate options compress file data +cmor.write(varid,values,len(time)) ; # Write variable with time axis +cmor.close() +f.close() +fc.close() +end_time = datetime.now() +print('done cmorizing ',mod, rn, vari,' process time: {}'.format(end_time-start_time)) + diff --git a/DataPreparationExamples/GARD-LENS/jobs.txt b/DataPreparationExamples/GARD-LENS/jobs.txt new file mode 100755 index 0000000..396a1b2 --- /dev/null +++ b/DataPreparationExamples/GARD-LENS/jobs.txt @@ -0,0 +1,22 @@ +python LOCA2_CMIP6_runCMOR.py ssp585 pr AWI-CM-1-1-MR +python LOCA2_CMIP6_runCMOR.py ssp585 pr IPSL-CM6A-LR +python LOCA2_CMIP6_runCMOR.py ssp585 pr ACCESS-CM2 +python LOCA2_CMIP6_runCMOR.py ssp585 pr INM-CM5-0 +python LOCA2_CMIP6_runCMOR.py ssp585 pr KACE-1-0-G +python LOCA2_CMIP6_runCMOR.py ssp585 pr MPI-ESM1-2-LR +python LOCA2_CMIP6_runCMOR.py ssp585 pr NorESM2-LM +python LOCA2_CMIP6_runCMOR.py ssp585 pr CanESM5 +python LOCA2_CMIP6_runCMOR.py ssp585 pr GFDL-ESM4 +python LOCA2_CMIP6_runCMOR.py ssp585 pr ACCESS-ESM1-5 +python LOCA2_CMIP6_runCMOR.py ssp585 pr GFDL-CM4 +python LOCA2_CMIP6_runCMOR.py ssp585 pr INM-CM4-8 +python LOCA2_CMIP6_runCMOR.py ssp585 pr EC-Earth3 +python LOCA2_CMIP6_runCMOR.py ssp585 pr BCC-CSM2-MR +python LOCA2_CMIP6_runCMOR.py ssp585 pr MPI-ESM1-2-HR +python LOCA2_CMIP6_runCMOR.py ssp585 pr TaiESM1 +python LOCA2_CMIP6_runCMOR.py ssp585 pr EC-Earth3-Veg +python LOCA2_CMIP6_runCMOR.py ssp585 pr FGOALS-g3 +python LOCA2_CMIP6_runCMOR.py ssp585 pr MIROC6 +python LOCA2_CMIP6_runCMOR.py ssp585 pr NorESM2-MM +python LOCA2_CMIP6_runCMOR.py ssp585 pr MRI-ESM2-0 + From f145b822ac30d27b4df5ef612657ccef5955b942 Mon Sep 17 00:00:00 2001 From: Peter J Gleckler Date: Tue, 8 Apr 2025 15:47:48 -0700 Subject: [PATCH 3/6] working again --- .../GARD-LENS/GARDLENS_CMIP6_input.json | 3 ++- .../GARD-LENS/GARDLENS_CMIP6_runCMOR.py | 7 ++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/DataPreparationExamples/GARD-LENS/GARDLENS_CMIP6_input.json b/DataPreparationExamples/GARD-LENS/GARDLENS_CMIP6_input.json index c9ad7a6..160da40 100644 --- a/DataPreparationExamples/GARD-LENS/GARDLENS_CMIP6_input.json +++ b/DataPreparationExamples/GARD-LENS/GARDLENS_CMIP6_input.json @@ -6,9 +6,10 @@ "source_id": "GARD-LENS-1-0", "institution_id":"NCAR", "driving_activity_id": "CMIP", - "driving_experiment_id": "historical", + "experiment_id": "historical", "driving_mip_era": "CMIP6", "driving_source_id": "ACCESS-CM2", "driving_variant_label": "r1i1p1f1", + "region":"NAM", "outpath":"/global/cfs/projectdirs/m3522/project_downscale/" } diff --git a/DataPreparationExamples/GARD-LENS/GARDLENS_CMIP6_runCMOR.py b/DataPreparationExamples/GARD-LENS/GARDLENS_CMIP6_runCMOR.py index 832ef0b..2e16de5 100644 --- a/DataPreparationExamples/GARD-LENS/GARDLENS_CMIP6_runCMOR.py +++ b/DataPreparationExamples/GARD-LENS/GARDLENS_CMIP6_runCMOR.py @@ -6,6 +6,11 @@ import sys, os, glob from datetime import datetime +# %% Get current script path, append src dir +current_dir = os.path.dirname(os.path.abspath(__file__)) +new_path = os.path.join(current_dir, "..", "..", "src") +sys.path.append(new_path) +from DRCDPLib import writeUserJson multi = False if multi == True: @@ -55,7 +60,7 @@ ##### CMOR setup cmor.setup(inpath='./',netcdf_file_action=cmor.CMOR_REPLACE_4,logfile= vari + '_' + mod + '-' + rn + '-'+'cmorLog.txt') -cmor.dataset_json(inputJson) +cmor.dataset_json(writeUserJson(inputJson, cmorTable)) cmor.load_table(cmorTable) # SET CMIP MODEL SPECIFIC ATTRIBUTES From 2bb4984f4a0e1e6d49e1978795bc515f5c055eed Mon Sep 17 00:00:00 2001 From: Peter J Gleckler Date: Tue, 8 Apr 2025 15:55:51 -0700 Subject: [PATCH 4/6] json streamlined --- .../GARD-LENS/GARDLENS_CMIP6_input.json | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/DataPreparationExamples/GARD-LENS/GARDLENS_CMIP6_input.json b/DataPreparationExamples/GARD-LENS/GARDLENS_CMIP6_input.json index 160da40..7f248fe 100644 --- a/DataPreparationExamples/GARD-LENS/GARDLENS_CMIP6_input.json +++ b/DataPreparationExamples/GARD-LENS/GARDLENS_CMIP6_input.json @@ -1,15 +1,10 @@ { - "_AXIS_ENTRY_FILE":"DRCDP_coordinate.json", - "_FORMULA_VAR_FILE":"DRCDP_formula_terms.json", - "_controlled_vocabulary_file":"DRCDP_CV.json", - "activity_id": "DRCDP", + "activity_id": "DRCDP", "source_id": "GARD-LENS-1-0", - "institution_id":"NCAR", "driving_activity_id": "CMIP", - "experiment_id": "historical", + "driving_experiment_id": "historical", "driving_mip_era": "CMIP6", "driving_source_id": "ACCESS-CM2", "driving_variant_label": "r1i1p1f1", - "region":"NAM", "outpath":"/global/cfs/projectdirs/m3522/project_downscale/" } From 0b960928a60a025cd0a560b994489ae89b1d0c18 Mon Sep 17 00:00:00 2001 From: Peter J Gleckler Date: Tue, 8 Apr 2025 16:32:59 -0700 Subject: [PATCH 5/6] setting up example --- .../GARD-LENS/GARDLENS_CMIP6_runCMOR.py | 39 +++++++---------- DataPreparationExamples/GARD-LENS/jobs.txt | 42 +++++++++---------- 2 files changed, 36 insertions(+), 45 deletions(-) diff --git a/DataPreparationExamples/GARD-LENS/GARDLENS_CMIP6_runCMOR.py b/DataPreparationExamples/GARD-LENS/GARDLENS_CMIP6_runCMOR.py index 2e16de5..7d9ca59 100644 --- a/DataPreparationExamples/GARD-LENS/GARDLENS_CMIP6_runCMOR.py +++ b/DataPreparationExamples/GARD-LENS/GARDLENS_CMIP6_runCMOR.py @@ -12,30 +12,27 @@ sys.path.append(new_path) from DRCDPLib import writeUserJson +### RUNNING PARALLEL VIA jobs.txt multi = False if multi == True: vari = sys.argv[1] domain = sys.argv[2] modi = sys.argv[3] rn = sys.argv[4] - - inputVarName = vari outputVarName = vari if vari == 'pr': outputUnits = 'kg m-2 s-1' if vari == 'tasmax': outputUnits = 'K' if vari == 'tasmin': outputUnits = 'K' - +# TEST vari = 'pcp' domain = 'CONUS' mod = 'ecearth3' rn = 'r150i1p1f1' - -inputVarName = vari outputVarName = 'pr' outputUnits = 'kg m-2 s-1' - +# CMOR TABLES cmorTable = '../../Tables/DRCDP_APday.json' inputJson = 'GARDLENS_CMIP6_input.json' @@ -44,19 +41,17 @@ start_time = datetime.now() #fc = xc.open_mfdataset(inFile,decode_times=True,use_cftime=True, preprocess=extract_date) fc = xc.open_dataset(inFile,decode_times=False,use_cftime=False) +fc = fc.isel(time=slice(0,100)) # test 100 time steps +d = fc[vari] -f = fc -f = fc.isel(time=slice(0,100)) -d = f[inputVarName] - -lat = f.lat.values -lon = f.lon.values -time = f.time.values -tunits = "days since 1900-01-01" +lat = fc.lat.values +lon = fc.lon.values +time = fc.time.values +tunits = fc.time.units -f = f.bounds.add_bounds("X") -f = f.bounds.add_bounds("Y") -f = f.bounds.add_bounds("T") +fc = fc.bounds.add_bounds("X") +fc = fc.bounds.add_bounds("Y") +fc = fc.bounds.add_bounds("T") ##### CMOR setup cmor.setup(inpath='./',netcdf_file_action=cmor.CMOR_REPLACE_4,logfile= vari + '_' + mod + '-' + rn + '-'+'cmorLog.txt') @@ -70,19 +65,15 @@ cmor.set_cur_dataset_attribute("driving_experiment_id",'historical') # Create CMOR axes -cmorLat = cmor.axis("latitude", coord_vals=lat[:], cell_bounds=f.lat_bnds.values, units="degrees_north") -cmorLon = cmor.axis("longitude", coord_vals=lon[:], cell_bounds=f.lon_bnds.values, units="degrees_east") -#cmorTime = cmor.axis("time", coord_vals=cftime.date2num(time,tunits), cell_bounds=cftime.date2num(f.time_bnds.values,tunits), units= tunits) -cmorTime = cmor.axis("time", coord_vals=time[:], cell_bounds=f.time_bnds.values[:], units= tunits) +cmorLat = cmor.axis("latitude", coord_vals=lat[:], cell_bounds=fc.lat_bnds.values, units="degrees_north") +cmorLon = cmor.axis("longitude", coord_vals=lon[:], cell_bounds=fc.lon_bnds.values, units="degrees_east") +cmorTime = cmor.axis("time", coord_vals=time[:], cell_bounds=fc.time_bnds.values[:], units= tunits) cmoraxes = [cmorTime,cmorLat, cmorLon] # Setup units and create variable to write using cmor - see https://cmor.llnl.gov/mydoc_cmor3_api/#cmor_set_variable_attribute varid = cmor.variable(outputVarName,outputUnits,cmoraxes,missing_value=1.e20) values = np.array(d[:],np.float32) -cmor.set_variable_attribute(varid,'valid_min','f',2.0) -cmor.set_variable_attribute(varid,'valid_max','f',3.0) - cmor.set_deflate(varid,1,1,1) ; # shuffle=1,deflate=1,deflate_level=1 - Deflate options compress file data cmor.write(varid,values,len(time)) ; # Write variable with time axis cmor.close() diff --git a/DataPreparationExamples/GARD-LENS/jobs.txt b/DataPreparationExamples/GARD-LENS/jobs.txt index 396a1b2..b2c3cd4 100755 --- a/DataPreparationExamples/GARD-LENS/jobs.txt +++ b/DataPreparationExamples/GARD-LENS/jobs.txt @@ -1,22 +1,22 @@ -python LOCA2_CMIP6_runCMOR.py ssp585 pr AWI-CM-1-1-MR -python LOCA2_CMIP6_runCMOR.py ssp585 pr IPSL-CM6A-LR -python LOCA2_CMIP6_runCMOR.py ssp585 pr ACCESS-CM2 -python LOCA2_CMIP6_runCMOR.py ssp585 pr INM-CM5-0 -python LOCA2_CMIP6_runCMOR.py ssp585 pr KACE-1-0-G -python LOCA2_CMIP6_runCMOR.py ssp585 pr MPI-ESM1-2-LR -python LOCA2_CMIP6_runCMOR.py ssp585 pr NorESM2-LM -python LOCA2_CMIP6_runCMOR.py ssp585 pr CanESM5 -python LOCA2_CMIP6_runCMOR.py ssp585 pr GFDL-ESM4 -python LOCA2_CMIP6_runCMOR.py ssp585 pr ACCESS-ESM1-5 -python LOCA2_CMIP6_runCMOR.py ssp585 pr GFDL-CM4 -python LOCA2_CMIP6_runCMOR.py ssp585 pr INM-CM4-8 -python LOCA2_CMIP6_runCMOR.py ssp585 pr EC-Earth3 -python LOCA2_CMIP6_runCMOR.py ssp585 pr BCC-CSM2-MR -python LOCA2_CMIP6_runCMOR.py ssp585 pr MPI-ESM1-2-HR -python LOCA2_CMIP6_runCMOR.py ssp585 pr TaiESM1 -python LOCA2_CMIP6_runCMOR.py ssp585 pr EC-Earth3-Veg -python LOCA2_CMIP6_runCMOR.py ssp585 pr FGOALS-g3 -python LOCA2_CMIP6_runCMOR.py ssp585 pr MIROC6 -python LOCA2_CMIP6_runCMOR.py ssp585 pr NorESM2-MM -python LOCA2_CMIP6_runCMOR.py ssp585 pr MRI-ESM2-0 +python GARDLENS_CMIP6_runCMOR.py ssp585 pr AWI-CM-1-1-MR +python GARDLENS_CMIP6_runCMOR.py ssp585 pr IPSL-CM6A-LR +python GARDLENS_CMIP6_runCMOR.py ssp585 pr ACCESS-CM2 +python GARDLENS_CMIP6_runCMOR.py ssp585 pr INM-CM5-0 +python GARDLENS_CMIP6_runCMOR.py ssp585 pr KACE-1-0-G +python GARDLENS_CMIP6_runCMOR.py ssp585 pr MPI-ESM1-2-LR +python GARDLENS_CMIP6_runCMOR.py ssp585 pr NorESM2-LM +python GARDLENS_CMIP6_runCMOR.py ssp585 pr CanESM5 +python GARDLENS_CMIP6_runCMOR.py ssp585 pr GFDL-ESM4 +python GARDLENS_CMIP6_runCMOR.py ssp585 pr ACCESS-ESM1-5 +python GARDLENS_CMIP6_runCMOR.py ssp585 pr GFDL-CM4 +python GARDLENS_CMIP6_runCMOR.py ssp585 pr INM-CM4-8 +python GARDLENS_CMIP6_runCMOR.py ssp585 pr EC-Earth3 +python GARDLENS_CMIP6_runCMOR.py ssp585 pr BCC-CSM2-MR +python GARDLENS_CMIP6_runCMOR.py ssp585 pr MPI-ESM1-2-HR +python GARDLENS_CMIP6_runCMOR.py ssp585 pr TaiESM1 +python GARDLENS_CMIP6_runCMOR.py ssp585 pr EC-Earth3-Veg +python GARDLENS_CMIP6_runCMOR.py ssp585 pr FGOALS-g3 +python GARDLENS_CMIP6_runCMOR.py ssp585 pr MIROC6 +python GARDLENS_CMIP6_runCMOR.py ssp585 pr NorESM2-MM +python GARDLENS_CMIP6_runCMOR.py ssp585 pr MRI-ESM2-0 From 55b395e52991161dd7f3a3d82faaf91a3527b6f7 Mon Sep 17 00:00:00 2001 From: Peter J Gleckler Date: Tue, 8 Apr 2025 16:41:25 -0700 Subject: [PATCH 6/6] preparing example --- DRCDP_source_id.json | 28 +++---------------- .../GARD-LENS/GARDLENS_CMIP6_runCMOR.py | 1 - 2 files changed, 4 insertions(+), 25 deletions(-) diff --git a/DRCDP_source_id.json b/DRCDP_source_id.json index 1294e4f..3d08f7a 100644 --- a/DRCDP_source_id.json +++ b/DRCDP_source_id.json @@ -12,33 +12,13 @@ "license_url":"https://creativecommons.org/licenses/by/4.0/", "nominal_resolution":"5 km", "product":"downscaled-statistical", - "reference":"Hartke, S.H., Newman, A.J., Gutmann, E. et al. GARD-LENS: A downscaled large ensemble dataset for understanding future climate and its uncertainties. Sci Data 11, 1374 (2024). https://doi.org/10.1038/s41597-024-04205-z", + "reference":" Hartke, Samantha. H., Andrew J. Newman, Ethan Gutmann, Rachel McCrary, Nicholas D. Lybarger, and Flavio Lehner (2024) GARD-LENS: A downscaled large ensemble dataset for understanding future climate and its uncertainties. Sci Data 11, 1374, 15 pp. https://doi.org/10.1038/s41597-024-04205-z", "region":"north_america", "region_id":"NAM", - "source":"GARD-LENS-1-0: Statistically-downscaled climate model projections based on CMIP6", + "source":"GARD-LENS 1.0: Statistically-downscaled climate model projections based on CMIP6", "source_name":"GARD-LENS", "source_version":"1.0", - "title":"GARD-LENS-1-0 dataset prepared for DRCDP" - }, - "GARD-LENS1-0":{ - "calendar":"standard", - "contact":"Samantha Hartke; hartke@ucar.edu", - "further_info_url":"https://opensky.ucar.edu/islandora/object/articles%3A42362", - "grid":"5 x 5 km latitude x longitude", - "grid_label":"gn", - "institution_id":"NCAR", - "license":"Creative Commons Attribution 4.0 International", - "license_id":"CC BY 4.0", - "license_url":"https://creativecommons.org/licenses/by/4.0/", - "nominal_resolution":"5 km", - "product":"downscaled-statistical", - "reference":"Hartke, S.H., Newman, A.J., Gutmann, E. et al. GARD-LENS: A downscaled large ensemble dataset for understanding future climate and its uncertainties. Sci Data 11, 1374 (2024). https://doi.org/10.1038/s41597-024-04205-z", - "region":"north_america", - "region_id":"NAM", - "source":"GARD-LENS-1-0: Statistically-downscaled climate model projections based on CMIP6", - "source_name":"GARD-LENS", - "source_version":"1.0", - "title":"GARD-LENS-1-0 dataset prepared for DRCDP" + "title":"GARD-LENS 1.0 dataset prepared for DRCDP" }, "LOCA2-0":{ "calendar":"gregorian", @@ -121,4 +101,4 @@ "title":"STAR-ESDM 1.0 dataset prepared for DRCDP" } } -} \ No newline at end of file +} diff --git a/DataPreparationExamples/GARD-LENS/GARDLENS_CMIP6_runCMOR.py b/DataPreparationExamples/GARD-LENS/GARDLENS_CMIP6_runCMOR.py index 7d9ca59..e9ec166 100644 --- a/DataPreparationExamples/GARD-LENS/GARDLENS_CMIP6_runCMOR.py +++ b/DataPreparationExamples/GARD-LENS/GARDLENS_CMIP6_runCMOR.py @@ -77,7 +77,6 @@ cmor.set_deflate(varid,1,1,1) ; # shuffle=1,deflate=1,deflate_level=1 - Deflate options compress file data cmor.write(varid,values,len(time)) ; # Write variable with time axis cmor.close() -f.close() fc.close() end_time = datetime.now() print('done cmorizing ',mod, rn, vari,' process time: {}'.format(end_time-start_time))