diff --git a/docs/examples/system-models/plot_oedi_9068.py b/docs/examples/system-models/plot_oedi_9068.py index 8544f34c9a..9a73bfa9ce 100644 --- a/docs/examples/system-models/plot_oedi_9068.py +++ b/docs/examples/system-models/plot_oedi_9068.py @@ -143,10 +143,12 @@ keys = ['ghi', 'dni', 'dhi', 'temp_air', 'wind_speed', 'albedo', 'precipitable_water'] -psm3, psm3_metadata = pvlib.iotools.get_psm3(latitude, longitude, api_key, - email, interval=5, names=2019, - map_variables=True, leap_day=True, - attributes=keys) +psm3, psm3_metadata = pvlib.iotools.get_nsrdb_psm4_conus(latitude, longitude, + api_key, email, + year=2019, interval=5, + parameters=keys, + map_variables=True, + leap_day=True) # %% # Pre-generate some model inputs diff --git a/docs/sphinx/source/reference/iotools.rst b/docs/sphinx/source/reference/iotools.rst index 12db7d6818..70a727b7bb 100644 --- a/docs/sphinx/source/reference/iotools.rst +++ b/docs/sphinx/source/reference/iotools.rst @@ -81,9 +81,6 @@ Satellite-derived irradiance and weather data for the Americas. iotools.get_nsrdb_psm4_conus iotools.get_nsrdb_psm4_full_disc iotools.read_nsrdb_psm4 - iotools.get_psm3 - iotools.read_psm3 - iotools.parse_psm3 Commercial datasets diff --git a/docs/sphinx/source/user_guide/extras/faq.rst b/docs/sphinx/source/user_guide/extras/faq.rst index f87fa101b5..29eeef7c2b 100644 --- a/docs/sphinx/source/user_guide/extras/faq.rst +++ b/docs/sphinx/source/user_guide/extras/faq.rst @@ -52,15 +52,7 @@ Where can I get irradiance data for my simulation? pvlib has a module called iotools which has several functions for retrieving irradiance data as well as reading standard file formats -such as EPW, TMY2, and TMY3. For free irradiance data, you may -consider NREL's NSRDB which can be accessed using the -:py:func:`pvlib.iotools.get_psm3` function and is available for -North America. For Europe and Africa, you may consider looking into -CAMS (:py:func:`pvlib.iotools.get_cams`). -PVGIS (:py:func:`pvlib.iotools.get_pvgis_hourly`) is another option, which -provides irradiance from several different databases with near global coverage. -pvlib also has functions for accessing a plethora of ground-measured -irradiance datasets, including the BSRN, SURFRAD, SRML, and NREL's MIDC. +such as EPW, TMY2, and TMY3. See :ref:`weatherdata`. Can I use PVsyst (PAN/OND) files with pvlib? diff --git a/docs/sphinx/source/user_guide/modeling_topics/weather_data.rst b/docs/sphinx/source/user_guide/modeling_topics/weather_data.rst index 8199044a5c..8e99895ab5 100644 --- a/docs/sphinx/source/user_guide/modeling_topics/weather_data.rst +++ b/docs/sphinx/source/user_guide/modeling_topics/weather_data.rst @@ -112,7 +112,8 @@ online web APIs. For example, :py:func:`~pvlib.iotools.get_pvgis_hourly` downloads data from PVGIS's webservers and returns it as a python variable. Functions that retrieve data from the internet are named ``get_``, followed by the name of the data source: :py:func:`~pvlib.iotools.get_bsrn`, -:py:func:`~pvlib.iotools.get_psm3`, :py:func:`~pvlib.iotools.get_pvgis_tmy`, +:py:func:`~pvlib.iotools.get_nsrdb_psm4_conus`, +:py:func:`~pvlib.iotools.get_pvgis_tmy`, and so on. For satellite/reanalysis datasets, the location is specified by latitude and @@ -121,7 +122,7 @@ longitude in decimal degrees: .. code-block:: python latitude, longitude = 33.75, -84.39 # Atlanta, Georgia, United States - df, metadata = pvlib.iotools.get_psm3(latitude, longitude, map_variables=True, ...) + df, metadata = pvlib.iotools.get_pvgis_tmy(latitude, longitude, map_variables=True, ...) For ground station networks, the location identifier is the station ID: diff --git a/docs/sphinx/source/whatsnew/v0.10.0.rst b/docs/sphinx/source/whatsnew/v0.10.0.rst index 6b6ae1abe9..eaefe12672 100644 --- a/docs/sphinx/source/whatsnew/v0.10.0.rst +++ b/docs/sphinx/source/whatsnew/v0.10.0.rst @@ -109,9 +109,9 @@ Enhancements :py:func:`pvlib.iotools.get_pvgis_horizon`. (:issue:`1290`, :pull:`1395`) * Update the URL used in the :py:func:`pvlib.iotools.get_cams` function. The new URL supports load-balancing and redirects to the fastest server. (:issue:`1688`, :pull:`1740`) -* :py:func:`pvlib.iotools.get_psm3` now has a ``url`` parameter to give the user +* :py:func:`!pvlib.iotools.get_psm3` now has a ``url`` parameter to give the user the option of controlling what NSRDB endpoint is used. (:pull:`1736`) -* :py:func:`pvlib.iotools.get_psm3` now uses the new NSRDB 3.2.2 endpoint for +* :py:func:`!pvlib.iotools.get_psm3` now uses the new NSRDB 3.2.2 endpoint for hourly and half-hourly single-year datasets. (:issue:`1591`, :pull:`1736`) * The default solar position algorithm (NREL SPA) is now 50-100% faster. (:pull:`1748`) * Added functions to retrieve daily precipitation, temperature, and snowfall data @@ -146,7 +146,7 @@ Testing Documentation ~~~~~~~~~~~~~ * Updated the description of the interval parameter in - :py:func:`pvlib.iotools.get_psm3`. (:issue:`1702`, :pull:`1712`) + :py:func:`!pvlib.iotools.get_psm3`. (:issue:`1702`, :pull:`1712`) * Fixed outdated nbviewer links. (:issue:`1721`, :pull:`1726`) diff --git a/docs/sphinx/source/whatsnew/v0.10.2.rst b/docs/sphinx/source/whatsnew/v0.10.2.rst index 3b82d98613..0bba4c69d1 100644 --- a/docs/sphinx/source/whatsnew/v0.10.2.rst +++ b/docs/sphinx/source/whatsnew/v0.10.2.rst @@ -28,7 +28,7 @@ Enhancements Bug fixes ~~~~~~~~~ -* :py:func:`~pvlib.iotools.get_psm3` no longer incorrectly returns clear-sky +* :py:func:`!pvlib.iotools.get_psm3` no longer incorrectly returns clear-sky DHI instead of clear-sky GHI when requesting ``ghi_clear``. (:pull:`1819`) * :py:func:`pvlib.singlediode.bishop88` with ``method='newton'`` no longer crashes when passed ``pandas.Series`` of length one. diff --git a/docs/sphinx/source/whatsnew/v0.10.3.rst b/docs/sphinx/source/whatsnew/v0.10.3.rst index 4d222fca06..6e4ac8e0ca 100644 --- a/docs/sphinx/source/whatsnew/v0.10.3.rst +++ b/docs/sphinx/source/whatsnew/v0.10.3.rst @@ -29,7 +29,7 @@ Bug fixes * Fixed CAMS error message handler in :py:func:`pvlib.iotools.get_cams`. (:issue:`1799`, :pull:`1905`) * Fix mapping of the dew point column to ``temp_dew`` when ``map_variables`` - is True in :py:func:`pvlib.iotools.get_psm3`. (:pull:`1920`) + is True in :py:func:`!pvlib.iotools.get_psm3`. (:pull:`1920`) * Fix :py:class:`pvlib.modelchain.ModelChain` to use attribute `clearsky_model`. (:pull:`1924`) diff --git a/docs/sphinx/source/whatsnew/v0.11.0.rst b/docs/sphinx/source/whatsnew/v0.11.0.rst index 219b57a059..d71fdcd9d9 100644 --- a/docs/sphinx/source/whatsnew/v0.11.0.rst +++ b/docs/sphinx/source/whatsnew/v0.11.0.rst @@ -14,10 +14,10 @@ Breaking changes * ``pvlib.iotools.read_srml_month_from_solardat`` was deprecated in v0.10.0 and has now been completely removed. The function is replaced by :py:func:`~pvlib.iotools.get_srml()`. (:pull:`1779`, :pull:`1989`) -* The ``leap_day`` parameter in :py:func:`~pvlib.iotools.get_psm3` +* The ``leap_day`` parameter in :py:func:`!pvlib.iotools.get_psm3` now defaults to True instead of False. (:issue:`1481`, :pull:`1991`) -* :py:func:`~pvlib.iotools.get_psm3`, :py:func:`~pvlib.iotools.read_psm3`, and - :py:func:`~pvlib.iotools.parse_psm3` all now have ``map_variables=True`` by +* :py:func:`!pvlib.iotools.get_psm3`, :py:func:`!pvlib.iotools.read_psm3`, and + :py:func:`!pvlib.iotools.parse_psm3` all now have ``map_variables=True`` by default. (:issue:`1425`, :pull:`2094`) * The deprecated ``ivcurve_pnts`` parameter of :py:func:`pvlib.pvsystem.singlediode` is removed. Use :py:func:`pvlib.pvsystem.v_from_i` and diff --git a/docs/sphinx/source/whatsnew/v0.13.0.rst b/docs/sphinx/source/whatsnew/v0.13.0.rst index 754d2a539b..94b7aac356 100644 --- a/docs/sphinx/source/whatsnew/v0.13.0.rst +++ b/docs/sphinx/source/whatsnew/v0.13.0.rst @@ -28,7 +28,7 @@ Deprecations :pull:`2467`, :pull:`2466`) - :py:func:`~pvlib.iotools.parse_epw` - - :py:func:`~pvlib.iotools.parse_psm3` + - :py:func:`!pvlib.iotools.parse_psm3` - :py:func:`~pvlib.iotools.parse_cams` - :py:func:`~pvlib.iotools.parse_bsrn` diff --git a/docs/sphinx/source/whatsnew/v0.13.2.rst b/docs/sphinx/source/whatsnew/v0.13.2.rst index e12f7277e5..0956c74a44 100644 --- a/docs/sphinx/source/whatsnew/v0.13.2.rst +++ b/docs/sphinx/source/whatsnew/v0.13.2.rst @@ -6,6 +6,9 @@ v0.13.2 (Anticipated December, 2025) Breaking Changes ~~~~~~~~~~~~~~~~ +* Following the removal of the NSRDB PSM3 API, the :func:`!pvlib.iotools.get_psm3`, + :func:`!pvlib.iotools.read_psm3`, and :func:`!pvlib.iotools.parse_psm3` + functions are removed. (:issue:`2581`, :pull:`2582`) Deprecations diff --git a/docs/sphinx/source/whatsnew/v0.7.0.rst b/docs/sphinx/source/whatsnew/v0.7.0.rst index 011852e513..41df67ac85 100644 --- a/docs/sphinx/source/whatsnew/v0.7.0.rst +++ b/docs/sphinx/source/whatsnew/v0.7.0.rst @@ -148,7 +148,7 @@ Enhancements diode model fitting function '6parsolve' from NREL's System Advisor Model. * Add :py:func:`~pvlib.ivtools.fit_sdm_desoto`, a method to fit the De Soto single diode model to the typical specifications given in manufacturers datasheets. -* Add `timeout` to :py:func:`pvlib.iotools.get_psm3`. +* Add `timeout` to :py:func:`!pvlib.iotools.get_psm3`. * Add :py:func:`~pvlib.scaling.wvm`, a port of the wavelet variability model for computing reductions in variability due to a spatially distributed plant. * Add :py:meth:`~pvlib.location.Location.from_epw`, a method to create a Location diff --git a/docs/sphinx/source/whatsnew/v0.7.1.rst b/docs/sphinx/source/whatsnew/v0.7.1.rst index af0368821d..630da2cff1 100644 --- a/docs/sphinx/source/whatsnew/v0.7.1.rst +++ b/docs/sphinx/source/whatsnew/v0.7.1.rst @@ -5,8 +5,8 @@ v0.7.1 (January 17, 2020) Enhancements ~~~~~~~~~~~~ -* Added :py:func:`~pvlib.iotools.read_psm3` to read local NSRDB PSM3 files and - :py:func:`~pvlib.iotools.parse_psm3` to parse local NSRDB PSM3 file-like +* Added :py:func:`!pvlib.iotools.read_psm3` to read local NSRDB PSM3 files and + :py:func:`!pvlib.iotools.parse_psm3` to parse local NSRDB PSM3 file-like objects. (:issue:`841`) * Added `leap_day` parameter to `iotools.get_psm3` instead of hardcoding it as False. diff --git a/docs/sphinx/source/whatsnew/v0.8.0.rst b/docs/sphinx/source/whatsnew/v0.8.0.rst index 86fb81574f..e322b9fb0c 100644 --- a/docs/sphinx/source/whatsnew/v0.8.0.rst +++ b/docs/sphinx/source/whatsnew/v0.8.0.rst @@ -19,7 +19,7 @@ Breaking changes * :py:func:`pvlib.iotools.read_tmy3` can now only read local data files because the NREL RREDC server hosting the TMY3 dataset has been retired. For - fetching TMY data from NREL servers, :py:func:`pvlib.iotools.get_psm3` is + fetching TMY data from NREL servers, :py:func:`!pvlib.iotools.get_psm3` is now recommended to retrieve newer PSM3 data over the older TMY3 data. (:issue:`996`) (:pull:`1004`) diff --git a/docs/sphinx/source/whatsnew/v0.8.1.rst b/docs/sphinx/source/whatsnew/v0.8.1.rst index 885973f786..f3ce66c5ff 100644 --- a/docs/sphinx/source/whatsnew/v0.8.1.rst +++ b/docs/sphinx/source/whatsnew/v0.8.1.rst @@ -24,7 +24,7 @@ Enhancements 10 minutes. (:pull:`1074`) * Added :py:func:`pvlib.inverter.sandia_multi` and :py:func:`pvlib.inverter.pvwatts_multi` for modeling inverters with multiple MPPTs (:issue:`457`, :pull:`1085`, :pull:`1106`) -* Added optional ``attributes`` parameter to :py:func:`pvlib.iotools.get_psm3` +* Added optional ``attributes`` parameter to :py:func:`!pvlib.iotools.get_psm3` and added the option of fetching 5- and 15-minute PSM3 data. (:pull:`1086`) * Added :py:func:`pvlib.irradiance.campbell_norman` for estimating DNI, DHI and GHI from extraterrestrial irradiance. This function replaces ``pvlib.irradiance.liujordan``; diff --git a/docs/sphinx/source/whatsnew/v0.9.0.rst b/docs/sphinx/source/whatsnew/v0.9.0.rst index d508f8871a..f574852126 100644 --- a/docs/sphinx/source/whatsnew/v0.9.0.rst +++ b/docs/sphinx/source/whatsnew/v0.9.0.rst @@ -50,7 +50,7 @@ Breaking changes :py:meth:`~pvlib.pvsystem.PVSystem.calcparams_cec` (:issue:`1118`, :pull:`1222`) * Switched the order of the outputs from the PSM3 iotools, notably - :py:func:`~pvlib.iotools.get_psm3` and :py:func:`~pvlib.iotools.read_psm3` + :py:func:`!pvlib.iotools.get_psm3` and :py:func:`!pvlib.iotools.read_psm3` (:issue:`1245`, :pull:`1268`) * Changed the naming of the inputs ``startdate``/``enddate`` to ``start``/``end`` in @@ -223,7 +223,7 @@ Documentation and to make the procedural and OO results match exactly. (:issue:`1116`, :pull:`1144`) * Add a gallery example showing how to appropriately use interval-averaged weather data for modeling. (:pull:`1152`) -* Update documentation links in :py:func:`pvlib.iotools.get_psm3` (:pull:`1169`) +* Update documentation links in :py:func:`!pvlib.iotools.get_psm3` (:pull:`1169`) * Use ``Mount`` classes in ``introtutorial`` and ``pvsystem`` docs pages (:pull:`1267`) * Clarified how statistics are calculated for :py:func:`pvlib.clearsky.detect_clearsky` (:issue:`1070`, :pull:`1243`) diff --git a/docs/sphinx/source/whatsnew/v0.9.1.rst b/docs/sphinx/source/whatsnew/v0.9.1.rst index 0a6a0d70f9..1734398c47 100644 --- a/docs/sphinx/source/whatsnew/v0.9.1.rst +++ b/docs/sphinx/source/whatsnew/v0.9.1.rst @@ -18,8 +18,8 @@ Deprecations Enhancements ~~~~~~~~~~~~ -* Added ``map_variables`` option to :py:func:`pvlib.iotools.get_psm3` and - :py:func:`pvlib.iotools.read_psm3` (:pull:`1374`) +* Added ``map_variables`` option to :py:func:`!pvlib.iotools.get_psm3` and + :py:func:`!pvlib.iotools.read_psm3` (:pull:`1374`) * Added ``pvlib.bifacial.infinite_sheds``, containing a model for irradiance on front and back surfaces of bifacial arrays. (:pull:`717`) * Added ``map_variables`` option to :func:`~pvlib.iotools.read_crn` (:pull:`1368`) diff --git a/docs/sphinx/source/whatsnew/v0.9.2.rst b/docs/sphinx/source/whatsnew/v0.9.2.rst index 2616734036..01612b2fad 100644 --- a/docs/sphinx/source/whatsnew/v0.9.2.rst +++ b/docs/sphinx/source/whatsnew/v0.9.2.rst @@ -35,7 +35,7 @@ Bug fixes timestamps as either 24:00 (which is the standard) as well as 00:00. Previously 00:00 timestamps would incorrectly be moved one day forward. (:pull:`1494`) -* :py:func:`pvlib.iotools.get_psm3` now raises a deprecation warning if +* :py:func:`!pvlib.iotools.get_psm3` now raises a deprecation warning if the ``leap_day`` parameter is not specified in a single-year request. Starting in pvlib 0.11.0 ``leap_day`` will default to True instead of False. (:issue:`1481`, :pull:`1511`) diff --git a/docs/sphinx/source/whatsnew/v0.9.5.rst b/docs/sphinx/source/whatsnew/v0.9.5.rst index 23766d566e..8d9c1b0aec 100644 --- a/docs/sphinx/source/whatsnew/v0.9.5.rst +++ b/docs/sphinx/source/whatsnew/v0.9.5.rst @@ -44,7 +44,7 @@ Bug fixes incorrect loss results for systems that are near the ground. (:issue:`1636`, :pull:`1653`) * Fixed incorrect mapping of requested parameters names when using - :py:func:`pvlib.iotools.get_psm3`. + :py:func:`!pvlib.iotools.get_psm3`. Also fixed the random reordering of the dataframe columns. (:issue:`1629`, :issue:`1647`, :pull:`1648`) * When using ``utc_time_range`` with :py:func:`pvlib.iotools.read_ecmwf_macc`, diff --git a/pvlib/iotools/__init__.py b/pvlib/iotools/__init__.py index 75663507f3..56bbe2cffb 100644 --- a/pvlib/iotools/__init__.py +++ b/pvlib/iotools/__init__.py @@ -8,9 +8,6 @@ from pvlib.iotools.crn import read_crn # noqa: F401 from pvlib.iotools.solrad import read_solrad # noqa: F401 from pvlib.iotools.solrad import get_solrad # noqa: F401 -from pvlib.iotools.psm3 import get_psm3 # noqa: F401 -from pvlib.iotools.psm3 import read_psm3 # noqa: F401 -from pvlib.iotools.psm3 import parse_psm3 # noqa: F401 from pvlib.iotools.psm4 import get_nsrdb_psm4_aggregated # noqa: F401 from pvlib.iotools.psm4 import get_nsrdb_psm4_tmy # noqa: F401 from pvlib.iotools.psm4 import get_nsrdb_psm4_conus # noqa: F401 diff --git a/pvlib/iotools/psm3.py b/pvlib/iotools/psm3.py deleted file mode 100644 index 184a4a7028..0000000000 --- a/pvlib/iotools/psm3.py +++ /dev/null @@ -1,365 +0,0 @@ -""" -Get PSM3 TMY -see https://developer.nrel.gov/docs/solar/nsrdb/psm3_data_download/ -""" - -import io -import requests -import pandas as pd -from json import JSONDecodeError -from pvlib._deprecation import deprecated -from pvlib import tools - -NSRDB_API_BASE = "https://developer.nrel.gov" -PSM_URL = NSRDB_API_BASE + "/api/nsrdb/v2/solar/psm3-2-2-download.csv" -TMY_URL = NSRDB_API_BASE + "/api/nsrdb/v2/solar/psm3-tmy-download.csv" -PSM5MIN_URL = NSRDB_API_BASE + "/api/nsrdb/v2/solar/psm3-5min-download.csv" - -ATTRIBUTES = ( - 'air_temperature', 'dew_point', 'dhi', 'dni', 'ghi', 'surface_albedo', - 'surface_pressure', 'wind_direction', 'wind_speed') -PVLIB_PYTHON = 'pvlib python' - -# Dictionary mapping PSM3 response names to pvlib names -VARIABLE_MAP = { - 'GHI': 'ghi', - 'DHI': 'dhi', - 'DNI': 'dni', - 'Clearsky GHI': 'ghi_clear', - 'Clearsky DHI': 'dhi_clear', - 'Clearsky DNI': 'dni_clear', - 'Solar Zenith Angle': 'solar_zenith', - 'Temperature': 'temp_air', - 'Dew Point': 'temp_dew', - 'Relative Humidity': 'relative_humidity', - 'Pressure': 'pressure', - 'Wind Speed': 'wind_speed', - 'Wind Direction': 'wind_direction', - 'Surface Albedo': 'albedo', - 'Precipitable Water': 'precipitable_water', -} - -# Dictionary mapping pvlib names to PSM3 request names -# Note, PSM3 uses different names for the same variables in the -# response and the request -REQUEST_VARIABLE_MAP = { - 'ghi': 'ghi', - 'dhi': 'dhi', - 'dni': 'dni', - 'ghi_clear': 'clearsky_ghi', - 'dhi_clear': 'clearsky_dhi', - 'dni_clear': 'clearsky_dni', - 'solar_zenith': 'solar_zenith_angle', - 'temp_air': 'air_temperature', - 'temp_dew': 'dew_point', - 'relative_humidity': 'relative_humidity', - 'pressure': 'surface_pressure', - 'wind_speed': 'wind_speed', - 'wind_direction': 'wind_direction', - 'albedo': 'surface_albedo', - 'precipitable_water': 'total_precipitable_water', -} - - -def get_psm3(latitude, longitude, api_key, email, names='tmy', interval=60, - attributes=ATTRIBUTES, leap_day=True, full_name=PVLIB_PYTHON, - affiliation=PVLIB_PYTHON, map_variables=True, url=None, - timeout=30): - """ - Retrieve NSRDB PSM3 timeseries weather data from the PSM3 API. The NSRDB - is described in [1]_ and the PSM3 API is described in [2]_, [3]_, and [4]_. - - .. versionchanged:: 0.9.0 - The function now returns a tuple where the first element is a dataframe - and the second element is a dictionary containing metadata. Previous - versions of this function had the return values switched. - - .. versionchanged:: 0.10.0 - The default endpoint for hourly single-year datasets is now v3.2.2. - The previous datasets can still be accessed (for now) by setting - the ``url`` parameter to the original API endpoint - (``"https://developer.nrel.gov/api/nsrdb/v2/solar/psm3-download.csv"``). - - Parameters - ---------- - latitude : float or int - in decimal degrees, between -90 and 90, north is positive - longitude : float or int - in decimal degrees, between -180 and 180, east is positive - api_key : str - NREL Developer Network API key - email : str - NREL API uses this to automatically communicate messages back - to the user only if necessary - names : str, default 'tmy' - PSM3 API parameter specifing year (e.g. ``2020``) or TMY variant - to download (e.g. ``'tmy'`` or ``'tgy-2019'``). The allowed values - update periodically, so consult the NSRDB references below for the - current set of options. - interval : int, {60, 5, 15, 30} - interval size in minutes, must be 5, 15, 30 or 60. Must be 60 for - typical year requests (i.e., tmy/tgy/tdy). - attributes : list of str, optional - meteorological fields to fetch. If not specified, defaults to - ``pvlib.iotools.psm3.ATTRIBUTES``. See references [2]_, [3]_, and [4]_ - for lists of available fields. Alternatively, pvlib names may also be - used (e.g. 'ghi' rather than 'GHI'); see :const:`REQUEST_VARIABLE_MAP`. - To retrieve all available fields, set ``attributes=[]``. - leap_day : bool, default : True - include leap day in the results. Only used for single-year requests - (i.e., it is ignored for tmy/tgy/tdy requests). - full_name : str, default 'pvlib python' - optional - affiliation : str, default 'pvlib python' - optional - map_variables : bool, default True - When true, renames columns of the Dataframe to pvlib variable names - where applicable. See variable :const:`VARIABLE_MAP`. - url : str, optional - API endpoint URL. If not specified, the endpoint is determined from - the ``names`` and ``interval`` parameters. - timeout : int, default 30 - time in seconds to wait for server response before timeout - - Returns - ------- - data : pandas.DataFrame - timeseries data from NREL PSM3 - metadata : dict - metadata from NREL PSM3 about the record, see - :func:`pvlib.iotools.read_psm3` for fields - - Raises - ------ - requests.HTTPError - if the request response status is not ok, then the ``'errors'`` field - from the JSON response or any error message in the content will be - raised as an exception, for example if the `api_key` was rejected or if - the coordinates were not found in the NSRDB - - Notes - ----- - The required NREL developer key, `api_key`, is available for free by - registering at the `NREL Developer Network `_. - - .. warning:: The "DEMO_KEY" `api_key` is severely rate limited and may - result in rejected requests. - - .. warning:: PSM3 is limited to data found in the NSRDB, please consult the - references below for locations with available data. Additionally, - querying data with < 30-minute resolution uses a different API endpoint - with fewer available fields (see [4]_). - - See Also - -------- - pvlib.iotools.read_psm3 - - References - ---------- - - .. [1] `NREL National Solar Radiation Database (NSRDB) - `_ - .. [2] `Physical Solar Model (PSM) v3.2.2 - `_ - .. [3] `Physical Solar Model (PSM) v3 TMY - `_ - .. [4] `Physical Solar Model (PSM) v3 - Five Minute Temporal Resolution - `_ - """ - # The well know text (WKT) representation of geometry notation is strict. - # A POINT object is a string with longitude first, then the latitude, with - # four decimals each, and exactly one space between them. - longitude = ('%9.4f' % longitude).strip() - latitude = ('%8.4f' % latitude).strip() - # TODO: make format_WKT(object_type, *args) in tools.py - - # convert to string to accomodate integer years being passed in - names = str(names) - - # convert pvlib names in attributes to psm3 convention - attributes = [REQUEST_VARIABLE_MAP.get(a, a) for a in attributes] - - # required query-string parameters for request to PSM3 API - params = { - 'api_key': api_key, - 'full_name': full_name, - 'email': email, - 'affiliation': affiliation, - 'reason': PVLIB_PYTHON, - 'mailing_list': 'false', - 'wkt': 'POINT(%s %s)' % (longitude, latitude), - 'names': names, - 'attributes': ','.join(attributes), - 'leap_day': str(leap_day).lower(), - 'utc': 'false', - 'interval': interval - } - # request CSV download from NREL PSM3 - if url is None: - # determine the endpoint that suits the user inputs - if any(prefix in names for prefix in ('tmy', 'tgy', 'tdy')): - url = TMY_URL - elif interval in (5, 15): - url = PSM5MIN_URL - else: - url = PSM_URL - - response = requests.get(url, params=params, timeout=timeout) - if not response.ok: - # if the API key is rejected, then the response status will be 403 - # Forbidden, and then the error is in the content and there is no JSON - try: - errors = response.json()['errors'] - except JSONDecodeError: - errors = response.content.decode('utf-8') - raise requests.HTTPError(errors, response=response) - # the CSV is in the response content as a UTF-8 bytestring - # to use pandas we need to create a file buffer from the response - fbuf = io.StringIO(response.content.decode('utf-8')) - return read_psm3(fbuf, map_variables) - - -def read_psm3(filename, map_variables=True): - """ - Read an NSRDB PSM3 weather file (formatted as SAM CSV). The NSRDB - is described in [1]_ and the SAM CSV format is described in [2]_. - - .. versionchanged:: 0.9.0 - The function now returns a tuple where the first element is a dataframe - and the second element is a dictionary containing metadata. Previous - versions of this function had the return values switched. - - Parameters - ---------- - filename: str, path-like, or buffer - Filename or in-memory buffer of a file containing data to read. - map_variables: bool, default True - When true, renames columns of the Dataframe to pvlib variable names - where applicable. See variable :const:`VARIABLE_MAP`. - - Returns - ------- - data : pandas.DataFrame - timeseries data from NREL PSM3 - metadata : dict - metadata from NREL PSM3 about the record, see notes for fields - - Notes - ----- - The return is a tuple with two items. The first item is a dataframe with - the PSM3 timeseries data. - - The second item is a dictionary with metadata from NREL PSM3 about the - record containing the following fields: - - * Source - * Location ID - * City - * State - * Country - * Latitude - * Longitude - * Time Zone - * Elevation - * Local Time Zone - * Clearsky DHI Units - * Clearsky DNI Units - * Clearsky GHI Units - * Dew Point Units - * DHI Units - * DNI Units - * GHI Units - * Solar Zenith Angle Units - * Temperature Units - * Pressure Units - * Relative Humidity Units - * Precipitable Water Units - * Wind Direction Units - * Wind Speed Units - * Cloud Type -15 - * Cloud Type 0 - * Cloud Type 1 - * Cloud Type 2 - * Cloud Type 3 - * Cloud Type 4 - * Cloud Type 5 - * Cloud Type 6 - * Cloud Type 7 - * Cloud Type 8 - * Cloud Type 9 - * Cloud Type 10 - * Cloud Type 11 - * Cloud Type 12 - * Fill Flag 0 - * Fill Flag 1 - * Fill Flag 2 - * Fill Flag 3 - * Fill Flag 4 - * Fill Flag 5 - * Surface Albedo Units - * Version - - Examples - -------- - >>> # Read a local PSM3 file: - >>> df, metadata = iotools.read_psm3("data.csv") # doctest: +SKIP - - >>> # Read a file object or an in-memory buffer: - >>> with open(filename, 'r') as f: # doctest: +SKIP - ... df, metadata = iotools.read_psm3(f) # doctest: +SKIP - - See Also - -------- - pvlib.iotools.get_psm3 - - References - ---------- - .. [1] `NREL National Solar Radiation Database (NSRDB) - `_ - .. [2] `Standard Time Series Data File Format - `_ - """ - with tools._file_context_manager(filename) as fbuf: - # The first 2 lines of the response are headers with metadata - metadata_fields = fbuf.readline().split(',') - metadata_values = fbuf.readline().split(',') - # get the column names so we can set the dtypes - columns = fbuf.readline().split(',') - columns[-1] = columns[-1].strip() # strip trailing newline - # Since the header has so many columns, excel saves blank cols in the - # data below the header lines. - columns = [col for col in columns if col != ''] - dtypes = dict.fromkeys(columns, float) # all floats except datevec - dtypes.update({'Year': int, 'Month': int, 'Day': int, 'Hour': int, - 'Minute': int, 'Cloud Type': int, 'Fill Flag': int}) - data = pd.read_csv( - fbuf, header=None, names=columns, usecols=columns, dtype=dtypes, - delimiter=',', lineterminator='\n') # skip carriage returns \r - - metadata_fields[-1] = metadata_fields[-1].strip() # trailing newline - metadata_values[-1] = metadata_values[-1].strip() # trailing newline - metadata = dict(zip(metadata_fields, metadata_values)) - # the response is all strings, so set some metadata types to numbers - metadata['Local Time Zone'] = int(metadata['Local Time Zone']) - metadata['Time Zone'] = int(metadata['Time Zone']) - metadata['Latitude'] = float(metadata['Latitude']) - metadata['Longitude'] = float(metadata['Longitude']) - metadata['Elevation'] = int(metadata['Elevation']) - - # the response 1st 5 columns are a date vector, convert to datetime - dtidx = pd.to_datetime(data[['Year', 'Month', 'Day', 'Hour', 'Minute']]) - # in USA all timezones are integers - tz = 'Etc/GMT%+d' % -metadata['Time Zone'] - data.index = pd.DatetimeIndex(dtidx).tz_localize(tz) - - if map_variables: - data = data.rename(columns=VARIABLE_MAP) - metadata['latitude'] = metadata.pop('Latitude') - metadata['longitude'] = metadata.pop('Longitude') - metadata['altitude'] = metadata.pop('Elevation') - - return data, metadata - - -parse_psm3 = deprecated(since="0.13.0", name="parse_psm3", - alternative="read_psm3")(read_psm3) diff --git a/pvlib/iotools/psm4.py b/pvlib/iotools/psm4.py index 44325364c7..ecab84fd21 100644 --- a/pvlib/iotools/psm4.py +++ b/pvlib/iotools/psm4.py @@ -714,7 +714,6 @@ def read_nsrdb_psm4(filename, map_variables=True): pvlib.iotools.get_nsrdb_psm4_tmy pvlib.iotools.get_nsrdb_psm4_conus pvlib.iotools.get_nsrdb_psm4_full_disc - pvlib.iotools.read_psm3 References ---------- diff --git a/tests/iotools/test_psm3.py b/tests/iotools/test_psm3.py deleted file mode 100644 index 39de06d234..0000000000 --- a/tests/iotools/test_psm3.py +++ /dev/null @@ -1,191 +0,0 @@ -""" -test iotools for PSM3 -""" - -from pvlib.iotools import psm3 -from tests.conftest import ( - TESTS_DATA_DIR, - RERUNS, - RERUNS_DELAY, - assert_index_equal, - nrel_api_key, -) -import numpy as np -import pandas as pd -import pytest -from requests import HTTPError -from io import StringIO - -from pvlib._deprecation import pvlibDeprecationWarning - - -TMY_TEST_DATA = TESTS_DATA_DIR / 'test_psm3_tmy-2017.csv' -YEAR_TEST_DATA = TESTS_DATA_DIR / 'test_psm3_2017.csv' -YEAR_TEST_DATA_5MIN = TESTS_DATA_DIR / 'test_psm3_2019_5min.csv' -MANUAL_TEST_DATA = TESTS_DATA_DIR / 'test_read_psm3.csv' -LATITUDE, LONGITUDE = 40.5137, -108.5449 -METADATA_FIELDS = [ - 'Source', 'Location ID', 'City', 'State', 'Country', 'Latitude', - 'Longitude', 'Time Zone', 'Elevation', 'Local Time Zone', - 'Dew Point Units', 'DHI Units', 'DNI Units', 'GHI Units', - 'Temperature Units', 'Pressure Units', 'Wind Direction Units', - 'Wind Speed Units', 'Surface Albedo Units', 'Version'] -PVLIB_EMAIL = 'pvlib-admin@googlegroups.com' - - -def assert_psm3_equal(data, metadata, expected): - """check consistency of PSM3 data""" - # check datevec columns - assert np.allclose(data.Year, expected.Year) - assert np.allclose(data.Month, expected.Month) - assert np.allclose(data.Day, expected.Day) - assert np.allclose(data.Hour, expected.Hour) - assert np.allclose(data.Minute, expected.Minute) - # check data columns - assert np.allclose(data.GHI, expected.GHI) - assert np.allclose(data.DNI, expected.DNI) - assert np.allclose(data.DHI, expected.DHI) - assert np.allclose(data.Temperature, expected.Temperature) - assert np.allclose(data.Pressure, expected.Pressure) - assert np.allclose(data['Dew Point'], expected['Dew Point']) - assert np.allclose(data['Surface Albedo'], expected['Surface Albedo']) - assert np.allclose(data['Wind Speed'], expected['Wind Speed']) - assert np.allclose(data['Wind Direction'], expected['Wind Direction']) - # check header - for mf in METADATA_FIELDS: - assert mf in metadata - # check timezone - assert (data.index.tzinfo.zone == 'Etc/GMT%+d' % -metadata['Time Zone']) - - -@pytest.mark.remote_data -@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY) -def test_get_psm3_tmy(nrel_api_key): - """test get_psm3 with a TMY""" - data, metadata = psm3.get_psm3(LATITUDE, LONGITUDE, nrel_api_key, - PVLIB_EMAIL, names='tmy-2017', - leap_day=False, map_variables=False) - expected = pd.read_csv(TMY_TEST_DATA) - assert_psm3_equal(data, metadata, expected) - - -@pytest.mark.remote_data -@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY) -def test_get_psm3_singleyear(nrel_api_key): - """test get_psm3 with a single year""" - data, metadata = psm3.get_psm3(LATITUDE, LONGITUDE, nrel_api_key, - PVLIB_EMAIL, names='2017', - leap_day=False, map_variables=False, - interval=30) - expected = pd.read_csv(YEAR_TEST_DATA) - assert_psm3_equal(data, metadata, expected) - - -@pytest.mark.remote_data -@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY) -def test_get_psm3_5min(nrel_api_key): - """test get_psm3 for 5-minute data""" - data, metadata = psm3.get_psm3(LATITUDE, LONGITUDE, nrel_api_key, - PVLIB_EMAIL, names='2019', interval=5, - leap_day=False, map_variables=False) - assert len(data) == 525600/5 - first_day = data.loc['2019-01-01'] - expected = pd.read_csv(YEAR_TEST_DATA_5MIN) - assert_psm3_equal(first_day, metadata, expected) - - -@pytest.mark.remote_data -@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY) -def test_get_psm3_check_leap_day(nrel_api_key): - data_2012, _ = psm3.get_psm3(LATITUDE, LONGITUDE, nrel_api_key, - PVLIB_EMAIL, names="2012", interval=60, - leap_day=True, map_variables=False) - assert len(data_2012) == (8760 + 24) - - -@pytest.mark.parametrize('latitude, longitude, api_key, names, interval', - [(LATITUDE, LONGITUDE, 'BAD', 'tmy-2017', 60), - (51, -5, nrel_api_key, 'tmy-2017', 60), - (LATITUDE, LONGITUDE, nrel_api_key, 'bad', 60), - (LATITUDE, LONGITUDE, nrel_api_key, '2017', 15), - ]) -@pytest.mark.remote_data -@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY) -def test_get_psm3_tmy_errors( - latitude, longitude, api_key, names, interval -): - """Test get_psm3() for multiple erroneous input scenarios. - - These scenarios include: - * Bad API key -> HTTP 403 forbidden because api_key is rejected - * Bad latitude/longitude -> Coordinates were not found in the NSRDB. - * Bad name -> Name is not one of the available options. - * Bad interval, single year -> Intervals can only be 30 or 60 minutes. - """ - with pytest.raises(HTTPError) as excinfo: - psm3.get_psm3(latitude, longitude, api_key, PVLIB_EMAIL, - names=names, interval=interval, leap_day=False, - map_variables=False) - # ensure the HTTPError caught isn't due to overuse of the API key - assert "OVER_RATE_LIMIT" not in str(excinfo.value) - - -@pytest.fixture -def io_input(request): - """file-like object for read_psm3""" - with MANUAL_TEST_DATA.open() as f: - data = f.read() - obj = StringIO(data) - return obj - - -def test_parse_psm3(io_input): - """test parse_psm3""" - with pytest.warns(pvlibDeprecationWarning, match='Use read_psm3 instead'): - data, metadata = psm3.parse_psm3(io_input, map_variables=False) - expected = pd.read_csv(YEAR_TEST_DATA) - assert_psm3_equal(data, metadata, expected) - - -def test_read_psm3(): - """test read_psm3""" - data, metadata = psm3.read_psm3(MANUAL_TEST_DATA, map_variables=False) - expected = pd.read_csv(YEAR_TEST_DATA) - assert_psm3_equal(data, metadata, expected) - - -def test_read_psm3_buffer(io_input): - data, metadata = psm3.read_psm3(io_input, map_variables=False) - expected = pd.read_csv(YEAR_TEST_DATA) - assert_psm3_equal(data, metadata, expected) - - -def test_read_psm3_map_variables(): - """test read_psm3 map_variables=True""" - data, metadata = psm3.read_psm3(MANUAL_TEST_DATA, map_variables=True) - columns_mapped = ['Year', 'Month', 'Day', 'Hour', 'Minute', 'dhi', 'ghi', - 'dni', 'ghi_clear', 'dhi_clear', 'dni_clear', - 'Cloud Type', 'temp_dew', 'solar_zenith', - 'Fill Flag', 'albedo', 'wind_speed', - 'wind_direction', 'precipitable_water', - 'relative_humidity', 'temp_air', 'pressure'] - data, metadata = psm3.read_psm3(MANUAL_TEST_DATA, map_variables=True) - assert_index_equal(data.columns, pd.Index(columns_mapped)) - - -@pytest.mark.remote_data -@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY) -def test_get_psm3_attribute_mapping(nrel_api_key): - """Test that pvlib names can be passed in as attributes and get correctly - reverse mapped to PSM3 names""" - data, meta = psm3.get_psm3(LATITUDE, LONGITUDE, nrel_api_key, PVLIB_EMAIL, - names=2019, interval=60, - attributes=['ghi', 'wind_speed'], - leap_day=False, map_variables=True) - # Check that columns are in the correct order (GH1647) - expected_columns = [ - 'Year', 'Month', 'Day', 'Hour', 'Minute', 'ghi', 'wind_speed'] - pd.testing.assert_index_equal(pd.Index(expected_columns), data.columns) - assert 'latitude' in meta.keys() - assert 'longitude' in meta.keys() - assert 'altitude' in meta.keys()