diff --git a/examples/NLDAS_Data_Exploration.ipynb b/examples/NLDAS_Data_Exploration.ipynb index 56a143b..db762d4 100644 --- a/examples/NLDAS_Data_Exploration.ipynb +++ b/examples/NLDAS_Data_Exploration.ipynb @@ -11,7 +11,17 @@ "- Downloads data file(s) from NASA\n", "- Show attribute statistics and visualizations\n", "- Do viz-related data cleaning\n", - "- Show (corrected) attribute statistics and visualizations" + "- Show (corrected) attribute statistics and visualizations\n", + "\n", + "### Setup Instructions:\n", + "1. Create *.netrc* file in home dir according to [GES DISC site instructions](https://disc.gsfc.nasa.gov/information/howto?title=How%20to%20Download%20Data%20Files%20from%20HTTP%20Service%20with%20wget)\n", + "2. Create environment, install notebook pkgs, enable extension:\n", + "```\n", + "conda env create -n elm python=2.7 # 2.7 needed for pynio\n", + "source activate elm\n", + "conda install -c conda-forge pycurl lxml holoviews\n", + "jupyter nbextension enable --py widgetsnbextension # This should report \"OK\"\n", + "```" ] }, { @@ -179,9 +189,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [] } @@ -202,7 +210,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython2", - "version": "2.7.13" + "version": "2.7.14" } }, "nbformat": 4, diff --git a/examples/example_utils.py b/examples/example_utils.py index a4be88b..8a357bd 100644 --- a/examples/example_utils.py +++ b/examples/example_utils.py @@ -12,38 +12,54 @@ import requests from six.moves.urllib.parse import urlparse -from six.moves import range +from six.moves import range, input from lxml import etree, html from ipywidgets import widgets, Layout from IPython.display import display, Javascript -from pydap.cas.urs import setup_session -session = setup_session( - os.environ.get('NLDAS_USERNAME') or raw_input('NLDAS Username: '), - os.environ.get('NLDAS_PASSWORD') or getpass.getpass('Password: ') -) - -def get_request(url): - import pycurl - from io import BytesIO - buffer = BytesIO() - c = pycurl.Curl() - c.setopt(c.URL, url) - c.setopt(c.WRITEDATA, buffer) - c.perform() - c.close() - return buffer.getvalue() +PYCURL = True + +if not PYCURL: + from pydap.cas.urs import setup_session + session = setup_session( + os.environ.get('NLDAS_USERNAME') or input('NLDAS Username: '), + os.environ.get('NLDAS_PASSWORD') or getpass.getpass('Password: ') + ) + +def get_request(url, outfpath=None): + global PYCURL + if PYCURL: + # outfpath must be set + import pycurl + from io import BytesIO + buffer = BytesIO() + c = pycurl.Curl() + c.setopt(c.URL, url) + c.setopt(c.WRITEDATA, buffer) + c.setopt(c.COOKIEJAR, '/tmp/cookie.jar') + c.setopt(c.NETRC, True) + c.setopt(c.FOLLOWLOCATION, True) + #c.setopt(c.REMOTE_NAME, outfpath) + c.perform() + c.close() + return buffer.getvalue() + resp = requests.get(url) + return resp.text def dl_file(url): data_fpath = urlparse(url).path.lstrip(os.sep) data_dpath = os.path.dirname(data_fpath) if not os.path.exists(data_fpath): - resp = session.get(url) if not os.path.isdir(data_dpath): os.makedirs(data_dpath) - with open(data_fpath, 'w') as outfp: - outfp.write(resp.content) + if PYCURL: + with open(data_fpath, 'w') as outfp: + outfp.write(get_request(url)) + else: + resp = session.get(url) + with open(data_fpath, 'w') as outfp: + outfp.write(resp.content) return data_fpath def dups_to_indexes(field_names):