diff --git a/DRCDP_institution_id.json b/DRCDP_institution_id.json index 363e810..728a0db 100644 --- a/DRCDP_institution_id.json +++ b/DRCDP_institution_id.json @@ -1,5 +1,9 @@ { "institution_id":{ + "NCAR":{ + "contact":"Samantha Hartke; hartke@ucar.edu", + "name":"National Center for Atmospheric Research, Boulder, CO, USA" + }, "TTU":{ "ROR":"0405mnx93", "URL":"https://www.ttu.edu/", diff --git a/DRCDP_source_id.json b/DRCDP_source_id.json index 6ea377a..3d08f7a 100644 --- a/DRCDP_source_id.json +++ b/DRCDP_source_id.json @@ -1,5 +1,25 @@ { "source_id":{ + "GARD-LENS-1-0":{ + "calendar":"standard", + "contact":"Samantha Hartke; hartke@ucar.edu", + "further_info_url":"https://opensky.ucar.edu/islandora/object/articles%3A42362", + "grid":"5 x 5 km latitude x longitude", + "grid_label":"gn", + "institution_id":"NCAR", + "license":"Creative Commons Attribution 4.0 International", + "license_id":"CC BY 4.0", + "license_url":"https://creativecommons.org/licenses/by/4.0/", + "nominal_resolution":"5 km", + "product":"downscaled-statistical", + "reference":" Hartke, Samantha. H., Andrew J. Newman, Ethan Gutmann, Rachel McCrary, Nicholas D. Lybarger, and Flavio Lehner (2024) GARD-LENS: A downscaled large ensemble dataset for understanding future climate and its uncertainties. Sci Data 11, 1374, 15 pp. https://doi.org/10.1038/s41597-024-04205-z", + "region":"north_america", + "region_id":"NAM", + "source":"GARD-LENS 1.0: Statistically-downscaled climate model projections based on CMIP6", + "source_name":"GARD-LENS", + "source_version":"1.0", + "title":"GARD-LENS 1.0 dataset prepared for DRCDP" + }, "LOCA2-0":{ "calendar":"gregorian", "contact":"Daniel Feldman; DRFeldman@lbl.gov", @@ -81,4 +101,4 @@ "title":"STAR-ESDM 1.0 dataset prepared for DRCDP" } } -} \ No newline at end of file +} diff --git a/DataPreparationExamples/GARD-LENS/GARDLENS_CMIP6_input.json b/DataPreparationExamples/GARD-LENS/GARDLENS_CMIP6_input.json new file mode 100644 index 0000000..7f248fe --- /dev/null +++ b/DataPreparationExamples/GARD-LENS/GARDLENS_CMIP6_input.json @@ -0,0 +1,10 @@ +{ + "activity_id": "DRCDP", + "source_id": "GARD-LENS-1-0", + "driving_activity_id": "CMIP", + "driving_experiment_id": "historical", + "driving_mip_era": "CMIP6", + "driving_source_id": "ACCESS-CM2", + "driving_variant_label": "r1i1p1f1", + "outpath":"/global/cfs/projectdirs/m3522/project_downscale/" +} diff --git a/DataPreparationExamples/GARD-LENS/GARDLENS_CMIP6_runCMOR.py b/DataPreparationExamples/GARD-LENS/GARDLENS_CMIP6_runCMOR.py new file mode 100644 index 0000000..e9ec166 --- /dev/null +++ b/DataPreparationExamples/GARD-LENS/GARDLENS_CMIP6_runCMOR.py @@ -0,0 +1,83 @@ +import cmor +import xcdat as xc +import xarray as xr +import cftime +import numpy as np +import sys, os, glob +from datetime import datetime + +# %% Get current script path, append src dir +current_dir = os.path.dirname(os.path.abspath(__file__)) +new_path = os.path.join(current_dir, "..", "..", "src") +sys.path.append(new_path) +from DRCDPLib import writeUserJson + +### RUNNING PARALLEL VIA jobs.txt +multi = False +if multi == True: + vari = sys.argv[1] + domain = sys.argv[2] + modi = sys.argv[3] + rn = sys.argv[4] + outputVarName = vari + if vari == 'pr': outputUnits = 'kg m-2 s-1' + if vari == 'tasmax': outputUnits = 'K' + if vari == 'tasmin': outputUnits = 'K' + +# TEST +vari = 'pcp' +domain = 'CONUS' +mod = 'ecearth3' +rn = 'r150i1p1f1' +outputVarName = 'pr' +outputUnits = 'kg m-2 s-1' + +# CMOR TABLES +cmorTable = '../../Tables/DRCDP_APday.json' +inputJson = 'GARDLENS_CMIP6_input.json' + +inFile = '/global/cfs/projectdirs/m2637/GARDLENS/' + vari + '/GARDLENS_' + mod + '_' + rn + '_' + vari + '_1970_2100_' + domain + '.nc' + +start_time = datetime.now() +#fc = xc.open_mfdataset(inFile,decode_times=True,use_cftime=True, preprocess=extract_date) +fc = xc.open_dataset(inFile,decode_times=False,use_cftime=False) +fc = fc.isel(time=slice(0,100)) # test 100 time steps +d = fc[vari] + +lat = fc.lat.values +lon = fc.lon.values +time = fc.time.values +tunits = fc.time.units + +fc = fc.bounds.add_bounds("X") +fc = fc.bounds.add_bounds("Y") +fc = fc.bounds.add_bounds("T") + +##### CMOR setup +cmor.setup(inpath='./',netcdf_file_action=cmor.CMOR_REPLACE_4,logfile= vari + '_' + mod + '-' + rn + '-'+'cmorLog.txt') +cmor.dataset_json(writeUserJson(inputJson, cmorTable)) +cmor.load_table(cmorTable) + +# SET CMIP MODEL SPECIFIC ATTRIBUTES +#cmor.set_cur_dataset_attribute("source_id","LOCA2--" + mod) +cmor.set_cur_dataset_attribute("driving_source_id",mod) +cmor.set_cur_dataset_attribute("driving_variant_label",rn) +cmor.set_cur_dataset_attribute("driving_experiment_id",'historical') + +# Create CMOR axes +cmorLat = cmor.axis("latitude", coord_vals=lat[:], cell_bounds=fc.lat_bnds.values, units="degrees_north") +cmorLon = cmor.axis("longitude", coord_vals=lon[:], cell_bounds=fc.lon_bnds.values, units="degrees_east") +cmorTime = cmor.axis("time", coord_vals=time[:], cell_bounds=fc.time_bnds.values[:], units= tunits) +cmoraxes = [cmorTime,cmorLat, cmorLon] + +# Setup units and create variable to write using cmor - see https://cmor.llnl.gov/mydoc_cmor3_api/#cmor_set_variable_attribute +varid = cmor.variable(outputVarName,outputUnits,cmoraxes,missing_value=1.e20) +values = np.array(d[:],np.float32) + +cmor.set_deflate(varid,1,1,1) ; # shuffle=1,deflate=1,deflate_level=1 - Deflate options compress file data +cmor.write(varid,values,len(time)) ; # Write variable with time axis +cmor.close() +fc.close() +end_time = datetime.now() +print('done cmorizing ',mod, rn, vari,' process time: {}'.format(end_time-start_time)) + diff --git a/DataPreparationExamples/GARD-LENS/jobs.txt b/DataPreparationExamples/GARD-LENS/jobs.txt new file mode 100755 index 0000000..b2c3cd4 --- /dev/null +++ b/DataPreparationExamples/GARD-LENS/jobs.txt @@ -0,0 +1,22 @@ +python GARDLENS_CMIP6_runCMOR.py ssp585 pr AWI-CM-1-1-MR +python GARDLENS_CMIP6_runCMOR.py ssp585 pr IPSL-CM6A-LR +python GARDLENS_CMIP6_runCMOR.py ssp585 pr ACCESS-CM2 +python GARDLENS_CMIP6_runCMOR.py ssp585 pr INM-CM5-0 +python GARDLENS_CMIP6_runCMOR.py ssp585 pr KACE-1-0-G +python GARDLENS_CMIP6_runCMOR.py ssp585 pr MPI-ESM1-2-LR +python GARDLENS_CMIP6_runCMOR.py ssp585 pr NorESM2-LM +python GARDLENS_CMIP6_runCMOR.py ssp585 pr CanESM5 +python GARDLENS_CMIP6_runCMOR.py ssp585 pr GFDL-ESM4 +python GARDLENS_CMIP6_runCMOR.py ssp585 pr ACCESS-ESM1-5 +python GARDLENS_CMIP6_runCMOR.py ssp585 pr GFDL-CM4 +python GARDLENS_CMIP6_runCMOR.py ssp585 pr INM-CM4-8 +python GARDLENS_CMIP6_runCMOR.py ssp585 pr EC-Earth3 +python GARDLENS_CMIP6_runCMOR.py ssp585 pr BCC-CSM2-MR +python GARDLENS_CMIP6_runCMOR.py ssp585 pr MPI-ESM1-2-HR +python GARDLENS_CMIP6_runCMOR.py ssp585 pr TaiESM1 +python GARDLENS_CMIP6_runCMOR.py ssp585 pr EC-Earth3-Veg +python GARDLENS_CMIP6_runCMOR.py ssp585 pr FGOALS-g3 +python GARDLENS_CMIP6_runCMOR.py ssp585 pr MIROC6 +python GARDLENS_CMIP6_runCMOR.py ssp585 pr NorESM2-MM +python GARDLENS_CMIP6_runCMOR.py ssp585 pr MRI-ESM2-0 + diff --git a/Tables/DRCDP_CV.json b/Tables/DRCDP_CV.json index ae8667c..e80d07c 100644 --- a/Tables/DRCDP_CV.json +++ b/Tables/DRCDP_CV.json @@ -14,6 +14,10 @@ "gn":"data reported on the analysis-native grid" }, "institution_id":{ + "NCAR":{ + "contact":"Samantha Hartke; hartke@ucar.edu", + "name":"National Center for Atmospheric Research, Boulder, CO, USA" + }, "TTU":{ "ROR":"0405mnx93", "URL":"https://www.ttu.edu/", @@ -102,6 +106,46 @@ "variable_id" ], "source_id":{ + "GARD-LENS-1-0":{ + "calendar":"standard", + "contact":"Samantha Hartke; hartke@ucar.edu", + "further_info_url":"https://opensky.ucar.edu/islandora/object/articles%3A42362", + "grid":"5 x 5 km latitude x longitude", + "grid_label":"gn", + "institution_id":"NCAR", + "license":"Creative Commons Attribution 4.0 International", + "license_id":"CC BY 4.0", + "license_url":"https://creativecommons.org/licenses/by/4.0/", + "nominal_resolution":"5 km", + "product":"downscaled-statistical", + "reference":"Hartke, S.H., Newman, A.J., Gutmann, E. et al. GARD-LENS: A downscaled large ensemble dataset for understanding future climate and its uncertainties. Sci Data 11, 1374 (2024). https://doi.org/10.1038/s41597-024-04205-z", + "region":"north_america", + "region_id":"NAM", + "source":"GARD-LENS-1-0: Statistically-downscaled climate model projections based on CMIP6", + "source_name":"GARD-LENS", + "source_version":"1.0", + "title":"GARD-LENS-1-0 dataset prepared for DRCDP" + }, + "GARD-LENS1-0":{ + "calendar":"standard", + "contact":"Samantha Hartke; hartke@ucar.edu", + "further_info_url":"https://opensky.ucar.edu/islandora/object/articles%3A42362", + "grid":"5 x 5 km latitude x longitude", + "grid_label":"gn", + "institution_id":"NCAR", + "license":"Creative Commons Attribution 4.0 International", + "license_id":"CC BY 4.0", + "license_url":"https://creativecommons.org/licenses/by/4.0/", + "nominal_resolution":"5 km", + "product":"downscaled-statistical", + "reference":"Hartke, S.H., Newman, A.J., Gutmann, E. et al. GARD-LENS: A downscaled large ensemble dataset for understanding future climate and its uncertainties. Sci Data 11, 1374 (2024). https://doi.org/10.1038/s41597-024-04205-z", + "region":"north_america", + "region_id":"NAM", + "source":"GARD-LENS-1-0: Statistically-downscaled climate model projections based on CMIP6", + "source_name":"GARD-LENS", + "source_version":"1.0", + "title":"GARD-LENS-1-0 dataset prepared for DRCDP" + }, "LOCA2-0":{ "calendar":"gregorian", "contact":"Daniel Feldman; DRFeldman@lbl.gov", diff --git a/src/createCVs.ipynb b/src/createCVs.ipynb index c54b3c4..b7b6705 100644 --- a/src/createCVs.ipynb +++ b/src/createCVs.ipynb @@ -349,7 +349,11 @@ "] = \"Texas Tech University, 2520 Broadway Avenue, Lubbock, TX 79409, USA\"\n", "institution_id[\"institution_id\"][\"TTU\"][\"ROR\"] = \"0405mnx93\"\n", "institution_id[\"institution_id\"][\"TTU\"][\"URL\"] = \"https://www.ttu.edu/\"\n", - "institution_id" + "institution_id\n", + "\n", + "institution_id[\"institution_id\"][\"NCAR\"] = {}\n", + "institution_id[\"institution_id\"][\"NCAR\"][\"name\"] = \"National Center for Atmospheric Research, Boulder, CO, USA\"\n", + "institution_id[\"institution_id\"][\"NCAR\"][\"\"] = \"hartke@ucar.edu\"" ] }, { @@ -877,6 +881,27 @@ "source_id[\"source_id\"][\"STAR-ESDM1-0\"][\n", " \"title\"\n", "] = \"STAR-ESDM 1.0 dataset prepared for DRCDP\"\n", + "\n", + "source_id[\"source_id\"][\"source_id[\"source_id\"][\"STAR-ESDM1-0\"][\"region\"] = \"north_america\"\n", + "source_id[\"source_id\"][\"STAR-ESDM1-0\"][\"region_id\"] = \"NAM\"\n", + "source_id[\"source_id\"][\"STAR-ESDM1-0\"][\n", + " \"source\"\n", + "] = \"STAR-ESDM 1.0: Statistically-downscaled climate model projections based on CMIP6\"\n", + "source_id[\"source_id\"][\"STAR-ESDM1-0\"][\"source_name\"] = \"STAR-ESDM\"\n", + "source_id[\"source_id\"][\"STAR-ESDM1-0\"][\"source_version\"] = \"1.0\"\n", + "source_id[\"source_id\"][\"STAR-ESDM1-0\"][\n", + " \"title\"\n", + "] = \"STAR-ESDM 1.0 dataset prepared for DRCDP1-0\"][\"region\"] = \"north_america\"\n", + "source_id[\"source_id\"][\"STAR-ESDM1-0\"][\"region_id\"] = \"NAM\"\n", + "source_id[\"source_id\"][\"STAR-ESDM1-0\"][\n", + " \"source\"\n", + "] = \"STAR-ESDM 1.0: Statistically-downscaled climate model projections based on CMIP6\"\n", + "source_id[\"source_id\"][\"STAR-ESDM1-0\"][\"source_name\"] = \"STAR-ESDM\"\n", + "source_id[\"source_id\"][\"STAR-ESDM1-0\"][\"source_version\"] = \"1.0\"\n", + "source_id[\"source_id\"][\"STAR-ESDM1-0\"][\n", + " \"title\"\n", + "] = \"STAR-ESDM 1.0 dataset prepared for DRCDP\"\n", + "\n", "source_id" ] }, @@ -1056,7 +1081,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "NERSC Python", "language": "python", "name": "python3" }, @@ -1070,7 +1095,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.13.2" + "version": "3.11.7" } }, "nbformat": 4, diff --git a/src/createCVs.py b/src/createCVs.py new file mode 100644 index 0000000..20f034c --- /dev/null +++ b/src/createCVs.py @@ -0,0 +1,62 @@ +import json + + +# read data from local (ensuring a pull has occurred to sync with main) +f = open('../DRCDP_institution_id.json','r') +institution_id = json.load(f) + +with open('../DRCDP_institution_id.json','r') as f: institution_id = json.load(f) +with open('../DRCDP_source_id.json','r') as f: source_id = json.load(f) + +# add new institution_id +key = 'NCAR' +institution_id['institution_id'][key] = {} +institution_id['institution_id'][key][ 'contact' ] = 'Samantha Hartke; hartke@ucar.edu' +institution_id['institution_id'][key][ 'name' ] = 'National Center for Atmospheric Research, Boulder, CO, USA' +#institution_id['institution_id'][key]['ROR'] = '00d9ah105' +#institution_id['institution_id'][key]['URL'] = 'https://www.climatologylab.org' + +# add new source_id +key = 'GARD-LENS-1-0' +source_id['source_id'][key] = {} +source_id['source_id'][key]['calendar'] = 'standard' +source_id['source_id'][key]['contact'] = 'Samantha Hartke; hartke@ucar.edu' +source_id['source_id'][key]['further_info_url'] = 'https://opensky.ucar.edu/islandora/object/articles%3A42362' +source_id['source_id'][key]['grid'] = '5 x 5 km latitude x longitude' +source_id['source_id'][key]['grid_label'] = 'gn' +source_id['source_id'][key]['institution_id'] = 'NCAR' +source_id['source_id'][key][ 'license' ] = 'Creative Commons Attribution 4.0 International' +source_id['source_id'][key]['license_id'] = 'CC BY 4.0' +source_id['source_id'][key][ 'license_url' ] = 'https://creativecommons.org/licenses/by/4.0/' +source_id['source_id'][key]['nominal_resolution'] = '5 km' +source_id['source_id'][key]['product'] = 'downscaled-statistical' +source_id['source_id'][key]['reference'] = ' '.join( + ['Hartke, S.H., Newman, A.J., Gutmann, E. et al. GARD-LENS: A downscaled large ensemble dataset for understanding future climate and its uncertainties. Sci Data 11, 1374 (2024). https://doi.org/10.1038/s41597-024-04205-z' ] +) +source_id['source_id'][key]['region'] = 'north_america' +source_id['source_id'][key]['region_id'] = 'NAM' +source_id['source_id'][key][ 'source' ] = 'GARD-LENS-1-0: Statistically-downscaled climate model projections based on CMIP6' +source_id['source_id'][key]['source_name'] = 'GARD-LENS' +source_id['source_id'][key]['source_version'] = '1.0' +source_id['source_id'][key]['title'] = 'GARD-LENS-1-0 dataset prepared for DRCDP' + +# overwrite existing files +# write files +outFile = ''.join(['../DRCDP_institution_id.json']) +with open(outFile, 'w') as f: + json.dump(institution_id, f, ensure_ascii=True, sort_keys=True, indent=4, separators=(',', ':')) +outFile = ''.join(['../DRCDP_source_id.json']) +with open(outFile, 'w') as f: + json.dump(source_id, f, ensure_ascii=True, sort_keys=True, indent=4, separators=(',', ':')) + +# overwrite DRCDP_CV.json with new entries +f = open('../Tables/DRCDP_CV.json','r') +CV = json.load(f) +CV['CV']['institution_id'] = institution_id['institution_id'] +CV['CV']['source_id'] = source_id['source_id'] + +# overwrite DRCDP_CV.json +outFile = ''.join(['../Tables/DRCDP_CV.json']) +with open(outFile, 'w') as f: + json.dump(CV, f, ensure_ascii=True, sort_keys=True, indent=4, separators=(',', ':')) + diff --git a/src/pullTable.ipynb b/src/pullTable.ipynb index 924fd4f..75578a0 100644 --- a/src/pullTable.ipynb +++ b/src/pullTable.ipynb @@ -383,7 +383,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "NERSC Python", "language": "python", "name": "python3" }, @@ -397,7 +397,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.13.2" + "version": "3.11.7" } }, "nbformat": 4,