diff --git a/CHANGELOG.md b/CHANGELOG.md index 149c23d..6057392 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,15 @@ +### 2026-01-22 v1.2.1 + +Enhancement: + +* Added check for expired API token where tokens are used + +Updates to align with NEON data updates: + +* Updates to variables file for frame file data products +* Update aquatic meteorological redirect to account for Dead Lake site decommissioning + + ### 2025-10-27 v1.2.0 Bug fixes: diff --git a/dist/neonutilities-1.2.1-py3-none-any.whl b/dist/neonutilities-1.2.1-py3-none-any.whl new file mode 100644 index 0000000..a62130e Binary files /dev/null and b/dist/neonutilities-1.2.1-py3-none-any.whl differ diff --git a/dist/neonutilities-1.2.1.tar.gz b/dist/neonutilities-1.2.1.tar.gz new file mode 100644 index 0000000..f43b2aa Binary files /dev/null and b/dist/neonutilities-1.2.1.tar.gz differ diff --git a/pyproject.toml b/pyproject.toml index 2a8ee26..073dee5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "neonutilities" -version = "1.2.0" +version = "1.2.1" authors = [ {name="Claire Lunch", email="clunch@battelleecology.org"}, {name="Bridget Hass", email="bhass@battelleecology.org"}, diff --git a/src/neonutilities/__init__.py b/src/neonutilities/__init__.py index 1acef85..61489c5 100644 --- a/src/neonutilities/__init__.py +++ b/src/neonutilities/__init__.py @@ -8,6 +8,7 @@ from .tabular_download import zips_by_product from .get_issue_log import get_issue_log from .read_table_neon import read_table_neon +from .helper_mods.api_helpers import token_date from .unzip_and_stack import ( stack_by_table, load_by_product, diff --git a/src/neonutilities/__resources__/frame_file_variables.csv b/src/neonutilities/__resources__/frame_file_variables.csv index 5e2be3b..1c1c854 100644 --- a/src/neonutilities/__resources__/frame_file_variables.csv +++ b/src/neonutilities/__resources__/frame_file_variables.csv @@ -1,75 +1,87 @@ -table,fieldName,description,dataType,units,downloadPkg,pubFormat -MCC,dnaSampleID,Identifier for DNA sample,string,NA,expanded,asIs -MCC,dnaSampleCode,Barcode of a DNA sample,string,NA,expanded,asIs -MCC,sequenceName,Name associated with the sequence,string,NA,expanded,asIs -MCC,taxonSequence,Sequence associated with the taxon,string,NA,expanded,asIs -MCC,completeTaxonomy,Full taxonomic hierarchy for identified organism,string,NA,expanded,asIs -MCC,domain,The scientific name of the domain in which the taxon is classified,string,NA,expanded,asIs -MCC,kingdom,The scientific name of the kingdom in which the taxon is classified,string,NA,expanded,asIs -MCC,phylum,The scientific name of the phylum or division in which the taxon is classified,string,NA,expanded,asIs -MCC,class,The scientific name of the class in which the taxon is classified,string,NA,expanded,asIs -MCC,order,The scientific name of the order in which the taxon is classified,string,NA,expanded,asIs -MCC,family,The scientific name of the family in which the taxon is classified,string,NA,expanded,asIs -MCC,genus,The scientific name of the genus in which the organism is classified,string,NA,expanded,asIs -MCC,specificEpithet,The specific epithet (second part of the species name) of the scientific name applied to the taxon,string,NA,expanded,asIs -MCC,scientificName,"Scientific name, associated with the taxonID. This is the name of the lowest level taxonomic rank that can be determined",string,NA,expanded,asIs -MCC,individualCount,Number of individuals of the same type,integer,NA,expanded,integer -MCC,fileName,"Name of file, including file extension",string,NA,expanded,asIs -MCT,dnaSampleID,Identifier for DNA sample,string,NA,expanded,asIs -MCT,dnaSampleCode,Barcode of a DNA sample,string,NA,expanded,asIs -MCT,sequenceName,Name associated with the sequence,string,NA,expanded,asIs -MCT,taxonSequence,Sequence associated with the taxon,string,NA,expanded,asIs -MCT,domain,The scientific name of the domain in which the taxon is classified,string,NA,expanded,asIs -MCT,kingdom,The scientific name of the kingdom in which the taxon is classified,string,NA,expanded,asIs -MCT,phylum,The scientific name of the phylum or division in which the taxon is classified,string,NA,expanded,asIs -MCT,class,The scientific name of the class in which the taxon is classified,string,NA,expanded,asIs -MCT,order,The scientific name of the order in which the taxon is classified,string,NA,expanded,asIs -MCT,family,The scientific name of the family in which the taxon is classified,string,NA,expanded,asIs -MCT,genus,The scientific name of the genus in which the organism is classified,string,NA,expanded,asIs -MCT,specificEpithet,The specific epithet (second part of the species name) of the scientific name applied to the taxon,string,NA,expanded,asIs -MCT,scientificName,"Scientific name, associated with the taxonID. This is the name of the lowest level taxonomic rank that can be determined",string,NA,expanded,asIs -MCT,individualCount,Number of individuals of the same type,integer,NA,expanded,integer -REA,hoboSampleID,Unique identifier for the HOBO conductivity logger file,string,NA,expanded,asIs -REA,hoboSampleCode,Barcode of the HOBO conductivity logger file,string,NA,expanded,asIs -REA,measurementNumber,The number of the measurement in a time series,integer,NA,expanded,integer -REA,dateTimeLogger,Local date and time returned by a field data logger,dateTime,NA,expanded,asIs -REA,lowRangeHobo,Conductivity returned from a hobo logger for the low range,real,microsiemensPerCentimeter,expanded,asIs -REA,fullRangeHobo,Conductivity from a hobo logger for the full range,real,microsiemensPerCentimeter,expanded,asIs -REA,waterTemp,Temperature of water (C),real,celsius,expanded,asIs -REA,fullRangeSpCondLinear,Specific conductance calculated using linear method and fullRangeHobo,real,microsiemensPerCentimeter,expanded,asIs -REA,lowRangeSpCondLinear,Specific conductance calculated using linear method and lowRangeHobo,real,microsiemensPerCentimeter,expanded,asIs -REA,fullRangeSpCondNonlinear,Specific conductance calculated using non-linear method and fullRangeHobo,real,microsiemensPerCentimeter,expanded,asIs -REA,lowRangeSpCondNonlinear,Specific conductance calculated using non-linear method and lowRangeHobo,real,microsiemensPerCentimeter,expanded,asIs -REA,fileName,"Name of file, including file extension",string,NA,expanded,asIs -FSP,spectralSampleID,Identifier for a spectral sample,string,NA,expanded,asIs -FSP,spectralSampleCode,Barcode of a spectral sample,string,NA,expanded,asIs -FSP,wavelength,Wavelength of measurement,real,nanometer,expanded,asIs -FSP,reflectanceCondition,Conditions under which reflectance measurement was made,string,NA,expanded,asIs -FSP,reflectance,Reflectance of sample,real,proportion,expanded,asIs -FSP,fileName,"Name of file, including file extension",string,NA,expanded,asIs -BAT,siteID,NEON site code,string,NA,expanded,asIs -BAT,surveyEndDate,The date-time indicating the end of all surveying activities,dateTime,NA,expanded,asIs -BAT,eventID,"An identifier for the set of information associated with the event, which includes information about the place and time of the event",string,NA,expanded,asIs -BAT,collectDateTime,Date and time of the collection event,dateTime,NA,expanded,asIs -BAT,decimalLatitude,"The geographic latitude (in decimal degrees, WGS84) of the geographic center of the reference area",real,decimalDegree,expanded,asIs -BAT,decimalLongitude,"The geographic longitude (in decimal degrees, WGS84) of the geographic center of the reference area",real,decimalDegree,expanded,asIs -BAT,gpsCoordinateQuality,GPS quality of ping nearest to interval midpoint with valid GPS fix,string,NA,expanded,asIs -BAT,elevation,Elevation (in meters) above sea level,real,meter,expanded,asIs -BAT,gpsElevationQuality,Elevation quality of ping nearest to interval midpoint with valid GPS fix,string,NA,expanded,asIs -BAT,dataFileName,"Name of file or folder containing data, including file extension",string,NA,expanded,asIs -BAT,transducerName,Name of the transducer used in the survey,string,NA,expanded,asIs -BAT,transducerNumber,Number of transducer in pinging sequence order,integer,NA,expanded,asIs -BAT,sonarReportNumber,Unique number identifying the grouping of pings per data file,integer,NA,expanded,asIs -BAT,firstPingNumber,Ping number of first ping in track,integer,NA,expanded,asIs -BAT,lastPingNumber,Ping number of last ping in track,integer,NA,expanded,asIs -BAT,bottomProcessingStatus,Explains if the bottom depth value was manually edited during processing,string,NA,expanded,asIs -BAT,waterBodyBottomDepth,Average depth below the water surface at the bottom of the water body measured by sonar,real,meter,expanded,asIs -BAT,plantProcessingStatus,Explains if the plant height value was manually edited during processing,string,NA,expanded,asIs -BAT,submergedPlantHeight,Average height of submerged vegetation from the water body bottom measured by sonar,real,meter,expanded,asIs -BAT,relativePlantCoverage,Relative coverage of submerged vegetation between the report number's pings,real,percent,expanded,asIs -BAT,habitatProcessingStatus,Input parameter quality for the bottom type analysis: invalid often means a very soft bottom substrate,string,NA,expanded,asIs -BAT,assignedBottomType,The assigned bottom type value determined by the maximum interpolated membership value,integer,NA,expanded,asIs -BAT,bottomType1Membership,Interpolated membership value for the first bottom type cluster,real,NA,expanded,asIs -BAT,bottomType2Membership,Interpolated membership value for the second bottom type cluster if applicable,real,NA,expanded,asIs -BAT,bottomType3Membership,Interpolated membership value for the third bottom type cluster if applicable,real,NA,expanded,asIs -BAT,bottomType4Membership,Interpolated membership value for the fourth bottom type cluster if applicable,real,NA,expanded,asIs \ No newline at end of file +"table","fieldName","description","dataType","units","downloadPkg","pubFormat" +"MCC","dnaSampleID","Identifier for DNA sample","string",NA,"expanded","asIs" +"MCC","dnaSampleCode","Barcode of a DNA sample","string",NA,"expanded","asIs" +"MCC","sequencerRunID","Identifier for the sequencing run","string",NA,"expanded","asIs" +"MCC","sequenceName","Name associated with the sequence","string",NA,"expanded","asIs" +"MCC","taxonSequence","Sequence associated with the taxon","string",NA,"expanded","asIs" +"MCC","completeTaxonomy","Full taxonomic hierarchy for identified organism","string",NA,"expanded","asIs" +"MCC","domain","The scientific name of the domain in which the taxon is classified","string",NA,"expanded","asIs" +"MCC","kingdom","The scientific name of the kingdom in which the taxon is classified","string",NA,"expanded","asIs" +"MCC","phylum","The scientific name of the phylum or division in which the taxon is classified","string",NA,"expanded","asIs" +"MCC","class","The scientific name of the class in which the taxon is classified","string",NA,"expanded","asIs" +"MCC","order","The scientific name of the order in which the taxon is classified","string",NA,"expanded","asIs" +"MCC","family","The scientific name of the family in which the taxon is classified","string",NA,"expanded","asIs" +"MCC","genus","The scientific name of the genus in which the organism is classified","string",NA,"expanded","asIs" +"MCC","specificEpithet","The specific epithet (second part of the species name) of the scientific name applied to the taxon","string",NA,"expanded","asIs" +"MCC","scientificName","Scientific name, associated with the taxonID. This is the name of the lowest level taxonomic rank that can be determined","string",NA,"expanded","asIs" +"MCC","individualCount","Number of individuals of the same type","integer",NA,"expanded","integer" +"MCT","dnaSampleID","Identifier for DNA sample","string",NA,"expanded","asIs" +"MCT","dnaSampleCode","Barcode of a DNA sample","string",NA,"expanded","asIs" +"MCT","sequencerRunID","Identifier for the sequencing run","string",NA,"expanded","asIs" +"MCT","sequenceName","Name associated with the sequence","string",NA,"expanded","asIs" +"MCT","taxonSequence","Sequence associated with the taxon","string",NA,"expanded","asIs" +"MCT","domain","The scientific name of the domain in which the taxon is classified","string",NA,"expanded","asIs" +"MCT","kingdom","The scientific name of the kingdom in which the taxon is classified","string",NA,"expanded","asIs" +"MCT","phylum","The scientific name of the phylum or division in which the taxon is classified","string",NA,"expanded","asIs" +"MCT","class","The scientific name of the class in which the taxon is classified","string",NA,"expanded","asIs" +"MCT","order","The scientific name of the order in which the taxon is classified","string",NA,"expanded","asIs" +"MCT","family","The scientific name of the family in which the taxon is classified","string",NA,"expanded","asIs" +"MCT","genus","The scientific name of the genus in which the organism is classified","string",NA,"expanded","asIs" +"MCT","specificEpithet","The specific epithet (second part of the species name) of the scientific name applied to the taxon","string",NA,"expanded","asIs" +"MCT","scientificName","Scientific name, associated with the taxonID. This is the name of the lowest level taxonomic rank that can be determined","string",NA,"expanded","asIs" +"MCT","otherScientificName","Other scientific name, not included in the NEON constrained taxon tables. This is the name of the lowest level taxonomic rank that can be determined","string",NA,"expanded","asIs" +"MCT","accession","A unique identifier assigned to a record in a database to allow for its retrieval and tracking over time","string",NA,"expanded","asIs" +"MCT","kingdomConfidence","The confidence level of the kingdom assignment","real","proportion","expanded","asIs" +"MCT","phylumConfidence","The confidence level of the phylum assignment","real","proportion","expanded","asIs" +"MCT","classConfidence","The confidence level of the class assignment","real","proportion","expanded","asIs" +"MCT","orderConfidence","The confidence level of the order assignment","real","proportion","expanded","asIs" +"MCT","familyConfidence","The confidence level of the family assignment","real","proportion","expanded","asIs" +"MCT","genusConfidence","The confidence level of the genus assignment","real","proportion","expanded","asIs" +"MCT","speciesConfidence","The confidence level of the species assignment","real","proportion","expanded","asIs" +"MCT","individualCount","Number of individuals of the same type","integer",NA,"expanded","integer" +"REA","hoboSampleID","Unique identifier for the HOBO conductivity logger file","string",NA,"expanded","asIs" +"REA","hoboSampleCode","Barcode of the HOBO conductivity logger file","string",NA,"expanded","asIs" +"REA","measurementNumber","The number of the measurement in a time series","integer",NA,"expanded","integer" +"REA","dateTimeLogger","Local date and time returned by a field data logger","dateTime",NA,"expanded","yyyy-MM-dd'T'HH:mm:ss.SSS'Z'" +"REA","lowRangeHobo","Conductivity returned from a hobo logger for the low range","real","microsiemensPerCentimeter","expanded","asIs" +"REA","fullRangeHobo","Conductivity from a hobo logger for the full range","real","microsiemensPerCentimeter","expanded","asIs" +"REA","waterTemp","Temperature of water (C)","real","celsius","expanded","asIs" +"REA","fullRangeSpCondLinear","Specific conductance calculated using linear method and fullRangeHobo","real","microsiemensPerCentimeter","expanded","asIs" +"REA","lowRangeSpCondLinear","Specific conductance calculated using linear method and lowRangeHobo","real","microsiemensPerCentimeter","expanded","asIs" +"REA","fullRangeSpCondNonlinear","Specific conductance calculated using non-linear method and fullRangeHobo","real","microsiemensPerCentimeter","expanded","asIs" +"REA","lowRangeSpCondNonlinear","Specific conductance calculated using non-linear method and lowRangeHobo","real","microsiemensPerCentimeter","expanded","asIs" +"FSP","spectralSampleID","Identifier for a spectral sample","string",NA,"expanded","asIs" +"FSP","spectralSampleCode","Barcode of a spectral sample","string",NA,"expanded","asIs" +"FSP","wavelength","Wavelength of measurement","real","nanometer","expanded","asIs" +"FSP","reflectanceCondition","Conditions under which reflectance measurement was made","string",NA,"expanded","asIs" +"FSP","reflectance","Reflectance of sample","real","proportion","expanded","asIs" +"BAT","fileName","Name of downloaded file","string",NA,"expanded","asIs" +"BAT","table","Name of data table","string",NA,"expanded","asIs" +"BAT","siteID","NEON site code","string",NA,"expanded","asIs" +"BAT","surveyEndDate","The date-time indicating the end of all surveying activities","dateTime",NA,"expanded","asIs" +"BAT","eventID","An identifier for the set of information associated with the event, which includes information about the place and time of the event","string",NA,"expanded","asIs" +"BAT","collectDateTime","Date and time of the collection event","dateTime",NA,"expanded","asIs" +"BAT","surveyPointLatitude","The geographic latitude of the geographic center of a survey specific location","real","decimalDegree","expanded","asIs" +"BAT","surveyPointLongitude","The geographic longitude of the geographic center of a survey specific location","real","decimalDegree","expanded","asIs" +"BAT","surveyPointCoordinateQuality","Quality of GPS ping nearest to interval midpoint with valid GPS fix","string",NA,"expanded","asIs" +"BAT","surveyPointCoordinateUncertainty","The horizontal distance from the given decimalLatitude and decimalLongitude describing the smallest circle containing the whole of the survey specific Location. Zero is not a valid value for this term","real",meter,"expanded","asIs" +"BAT","surveyPointElevation","Elevation (in meters) above sea level of a survey specific location","real","meter","expanded","asIs" +"BAT","surveyPointElevationQuality","Elevation quality of GPS ping nearest to interval midpoint with valid GPS fix","string",NA,"expanded","asIs" +"BAT","surveyPointElevationUncertainty","Uncertainty in elevation values (in meters) of a survey specific location","real",meter,"expanded","asIs" +"BAT","dataFileName","Name of file or folder containing data, including file extension","string",NA,"expanded","asIs" +"BAT","transducerName","Name of the transducer used in the survey","string",NA,"expanded","asIs" +"BAT","transducerNumber","Number of transducer in pinging sequence order","integer",NA,"expanded","asIs" +"BAT","sonarReportNumber","Unique number identifying the grouping of pings per data file","integer",NA,"expanded","asIs" +"BAT","firstPingNumber","Ping number of first ping in track","integer",NA,"expanded","asIs" +"BAT","lastPingNumber","Ping number of last ping in track","integer",NA,"expanded","asIs" +"BAT","bottomProcessingStatus","Explains if the bottom depth value was manually edited during processing","string",NA,"expanded","asIs" +"BAT","waterBodyBottomDepth","Average depth below the water surface at the bottom of the water body measured by sonar","real","meter","expanded","asIs" +"BAT","plantProcessingStatus","Explains if the plant height value was manually edited during processing","string",NA,"expanded","asIs" +"BAT","submergedPlantHeight","Average height of submerged vegetation from the water body bottom measured by sonar","real","meter","expanded","asIs" +"BAT","relativePlantCoverage","Relative coverage of submerged vegetation between the report number's pings","real","percent","expanded","asIs" +"BAT","habitatProcessingStatus","Input parameter quality for the bottom type analysis: invalid often means a very soft bottom substrate","string",NA,"expanded","asIs" +"BAT","assignedBottomType","The assigned bottom type value determined by the maximum interpolated membership value","integer",NA,"expanded","asIs" +"BAT","bottomType1Membership","Interpolated membership value for the first bottom type cluster","real",NA,"expanded","asIs" +"BAT","bottomType2Membership","Interpolated membership value for the second bottom type cluster if applicable","real",NA,"expanded","asIs" +"BAT","bottomType3Membership","Interpolated membership value for the third bottom type cluster if applicable","real",NA,"expanded","asIs" +"BAT","bottomType4Membership","Interpolated membership value for the fourth bottom type cluster if applicable","real",NA,"expanded","asIs" diff --git a/src/neonutilities/aop_download.py b/src/neonutilities/aop_download.py index ef5b047..fa8283d 100644 --- a/src/neonutilities/aop_download.py +++ b/src/neonutilities/aop_download.py @@ -37,6 +37,7 @@ # local imports from . import __resources__ from .helper_mods.api_helpers import get_api +from .helper_mods.api_helpers import token_check from .helper_mods.api_helpers import download_file, calculate_crc32c from .helper_mods.metadata_helpers import convert_byte_size from .get_issue_log import get_issue_log @@ -875,6 +876,10 @@ def by_file_aop( if token == "": token = None + # check for expired token + if token is not None: + token = token_check(token) + # query the products endpoint for the product requested response = get_api("https://data.neonscience.org/api/v0/products/" + dpid, token) @@ -1395,6 +1400,10 @@ def by_tile_aop( if token == "": token = None + # check for expired token + if token is not None: + token = token_check(token) + # query the products endpoint for the product requested response = get_api("https://data.neonscience.org/api/v0/products/" + dpid, token) diff --git a/src/neonutilities/helper_mods/__init__.py b/src/neonutilities/helper_mods/__init__.py index 5615b36..9524475 100644 --- a/src/neonutilities/helper_mods/__init__.py +++ b/src/neonutilities/helper_mods/__init__.py @@ -3,4 +3,6 @@ from .api_helpers import get_zip_urls from .api_helpers import download_urls from .api_helpers import download_file +from .api_helpers import token_date +from .api_helpers import token_check from .metadata_helpers import get_recent diff --git a/src/neonutilities/helper_mods/api_helpers.py b/src/neonutilities/helper_mods/api_helpers.py index c09d1eb..211c026 100644 --- a/src/neonutilities/helper_mods/api_helpers.py +++ b/src/neonutilities/helper_mods/api_helpers.py @@ -7,6 +7,8 @@ import re import os import time +import base64 +import ast import platform import importlib.metadata import logging @@ -25,15 +27,83 @@ usera = f"neonutilities/{vers} Python/{plat} {osplat}" +def token_date(token, rval="string"): + """ + + Find the expiration date of a user-specific API token generated within data.neonscience.org user accounts. + + Parameters + -------- + token: User specific API token (generated within data.neonscience.org user accounts). + rval: Should returned value be a string or a time value? Defaults to string. + + Return + -------- + Date of expiration in local time + + Created on Jan 13 2026 + + @author: Claire Lunch + """ + + splittoken = token.split(".") + # + "===" added here as "padding" required to make all tokens interpretable + dubsplit = base64.urlsafe_b64decode(splittoken[1] + "===") + dictsplit = ast.literal_eval(dubsplit.decode("utf-8")) + if "exp" in dictsplit.keys(): + expsplit = dictsplit["exp"] + if rval=="string": + expdate = time.asctime(time.localtime(expsplit)) + return(expdate) + else: + return(expsplit) + else: + logging.info("No expiration date found for API token.") + return None + + +def token_check(token): + """ + + Check whether a NEON API token has expired. + + Parameters + -------- + token: User specific API token (generated within data.neonscience.org user accounts). + + Return + -------- + The original token, if unexpired. If expired, return None. + + Created on Jan 13 2026 + + @author: Claire Lunch + """ + + try: + expdate = token_date(token, rval="time") + except Exception: + logging.info("API token expiration date could not be determined.") + return(token) + + if(expdate is None): + return(token) + else: + if time.time() > expdate: + logging.info("API token has expired. Function will proceed using public access rate. Go to your NEON user account to generate a new token.") + token = None + return(token) + + def get_api(api_url, token=None): """ - Accesses the API with options to use the user-specific API token generated within neon.datascience user accounts. + Accesses the API with options to use the user-specific API token generated within data.neonscience.org user accounts. Parameters -------- api_url: The API endpoint URL. - token: User specific API token (generated within neon.datascience user accounts). Optional. + token: User specific API token (generated within data.neonscience.org user accounts). Optional. Return -------- diff --git a/src/neonutilities/read_table_neon.py b/src/neonutilities/read_table_neon.py index 2658984..f21f734 100644 --- a/src/neonutilities/read_table_neon.py +++ b/src/neonutilities/read_table_neon.py @@ -54,13 +54,16 @@ def get_variables(v): ]: typ = pa.timestamp("s", tz="UTC") else: - if v.pubFormat[i] in ["yyyy-MM-dd(floor)", "yyyy-MM-dd"]: - typ = pa.date64() + if v.pubFormat[i] in ["yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"]: + typ = pa.timestamp("ms", tz="UTC") else: - if v.pubFormat[i] in ["yyyy(floor)", "yyyy(round)"]: - typ = pa.int64() + if v.pubFormat[i] in ["yyyy-MM-dd(floor)", "yyyy-MM-dd"]: + typ = pa.date64() else: - typ = pa.string() + if v.pubFormat[i] in ["yyyy(floor)", "yyyy(round)"]: + typ = pa.int64() + else: + typ = pa.string() if i == 0: vschema = pa.schema([(nm, typ)]) else: diff --git a/src/neonutilities/tabular_download.py b/src/neonutilities/tabular_download.py index 5afac02..f03da2a 100644 --- a/src/neonutilities/tabular_download.py +++ b/src/neonutilities/tabular_download.py @@ -7,6 +7,7 @@ import pandas as pd import logging from .helper_mods.api_helpers import get_api +from .helper_mods.api_helpers import token_check from .helper_mods.api_helpers import get_zip_urls from .helper_mods.api_helpers import get_tab_urls from .helper_mods.api_helpers import download_urls @@ -57,6 +58,10 @@ def query_files( adict = lst["data"]["siteCodes"] releasedict = {} + + # check for expired token + if token is not None: + token = token_check(token) # check expanded package status if package == "expanded": @@ -372,18 +377,28 @@ def zips_by_product( indx = 0 for s in site: if s in shared_aquatic_df.index: - ss = shared_aquatic_df.loc[s] - if dpid in list(ss["product"]): + if s in ["BLWA"] and release not in ["RELEASE-2021","RELEASE-2022","RELEASE-2023","RELEASE-2024","RELEASE-2025"]: indx = indx + 1 - sx = list(ss["towerSite"][ss["product"] == dpid]) - siter.append(sx) + if "DELA" not in site: + siter.append(["DELA"]) if indx == 1: logging.info( f"Some NEON sites in your download request are aquatic sites where {dpid} is collected at a nearby terrestrial site. The sites you requested, and the sites that will be accessed instead, are listed below." - ) - logging.info(f"{s} -> {''.join(sx)}") + ) + logging.info("Until the fall of 2025, meteorological data for BLWA were collected at DELA. Data collection at DELA ended in late 2025 and the meteorological station was relocated to BLWA. If your download request crosses this time period, data will be downloaded from each site for the time period when they are available.") else: - siter.append([s]) + ss = shared_aquatic_df.loc[s] + if dpid in list(ss["product"]): + indx = indx + 1 + sx = list(ss["towerSite"][ss["product"] == dpid]) + siter.append(sx) + if indx == 1: + logging.info( + f"Some NEON sites in your download request are aquatic sites where {dpid} is collected at a nearby terrestrial site. The sites you requested, and the sites that will be accessed instead, are listed below." + ) + logging.info(f"{s} -> {''.join(sx)}") + else: + siter.append([s]) else: siter.append([s]) siter = sum(siter, []) @@ -421,6 +436,10 @@ def zips_by_product( f"In all NEON releases after {bundle_release}, {''.join(dpid)} has been bundled with {''.join(newDPID)} and is not available independently. Please download {''.join(newDPID)}." ) + # check for expired token + if token is not None: + token = token_check(token) + # end of error and exception handling, start the work # query the /products endpoint for the product requested if release == "current" or release == "PROVISIONAL": @@ -488,7 +507,7 @@ def zips_by_product( fls = query_files( lst=avail, dpid=dpid, - site=site, + site=siter, startdate=startdate, enddate=enddate, package=package, diff --git a/src/neonutilities/unzip_and_stack.py b/src/neonutilities/unzip_and_stack.py index 18e44e8..8f933f8 100644 --- a/src/neonutilities/unzip_and_stack.py +++ b/src/neonutilities/unzip_and_stack.py @@ -1742,6 +1742,11 @@ def dataset_query( "Table name (tabl=) is a required input to this function." ) + if pubtype in ["TOS Data Product Type","TOS-structured TIS Data Product Type"]: + raise ValueError( + f"{dpid} is an observational data product. hor and ver are not valid inputs for this data product type." + ) + if pubtype in ["TIS Data Product Type","AIS Data Product Type"]: if dpid in ["DP4.00200.001", "DP1.00007.001","DP1.00010.001","DP1.00034.001","DP1.00035.001", diff --git a/tests/test_get_citation.py b/tests/test_get_citation.py index 992e90f..8b13dd3 100644 --- a/tests/test_get_citation.py +++ b/tests/test_get_citation.py @@ -12,6 +12,7 @@ # import required packages import os +from datetime import date from src.neonutilities.citation import get_citation # read in token from os.environ @@ -25,7 +26,9 @@ def test_get_citation_provisional(): Test that the get_citation() function returns the expected citation for provisional data """ cit = get_citation(dpid="DP1.10003.001", release="PROVISIONAL") - citexp = "@misc{DP1.10003.001/provisional,\n doi = {},\n url = {https://data.neonscience.org/data-products/DP1.10003.001},\n author = {{National Ecological Observatory Network (NEON)}},\n language = {en},\n title = {Breeding landbird point counts (DP1.10003.001)},\n publisher = {National Ecological Observatory Network (NEON)},\n year = {2025}\n}" + dt = date.today() + year = dt.year + citexp = "@misc{DP1.10003.001/provisional,\n doi = {},\n url = {https://data.neonscience.org/data-products/DP1.10003.001},\n author = {{National Ecological Observatory Network (NEON)}},\n language = {en},\n title = {Breeding landbird point counts (DP1.10003.001)},\n publisher = {National Ecological Observatory Network (NEON)},\n year = {" + str(year) + "}\n}" assert cit == citexp