NLDAS notebook updates - get rid of unicode literals

.travis.yml

@@ -4,12 +4,10 @@ dist: trusty
 
 env:
   global:
-    - EARTHIO_VERSION=master
-    - EARTHIO_INSTALL_METHOD="conda"
-    - EARTHIO_TEST_ENV=earth-test-env
+    - TEST_ENV=earth-test-env
     - ELM_EXAMPLE_DATA_PATH=/tmp/elm-data
-    - EARTHIO_CHANNEL_STR=" -c ioam -c conda-forge -c scitools/label/dev -c bioconda"
-
+    - INSTALL_CHANNELS=" -c elm -c elm/label/dev -c ioam -c conda-forge -c scitools/label/dev -c bioconda "
+    - ANACONDA_UPLOAD_USER=elm
   matrix:
     - PYTHON=3.6 NUMPY=1.12
     - PYTHON=3.5 NUMPY=1.11 TEST_DOCS=1
@@ -25,12 +23,12 @@ before_install:
 
 install:
   - MAKE_MINICONDA=1 ./build_elm_env.sh
-  - pushd docs
-  - ~/miniconda/bin/conda env create -f environment.yml -n ${EARTHIO_TEST_ENV}-docs
-  - source ~/miniconda/bin/activate ${EARTHIO_TEST_ENV}-docs
+  #- pushd docs
+  #- ~/miniconda/bin/conda env create -f environment.yml -n ${TEST_ENV}-docs
+  #- source ~/miniconda/bin/activate ${TEST_ENV}-docs
   # - if [ "$TEST_DOCS" ]; then conda install -c conda-forge -c ioam -c scitools --use-local elm earthio && make html && make doctest; fi
-  - source deactivate
-  - popd
+  #- source deactivate
+  #- popd
 
 script:
   - rm -rf $ELM_EXAMPLE_DATA_PATH/*
@@ -40,11 +38,11 @@ notifications:
   on_failure: always
   flowdock: $FD_TOKEN
 
-#deploy:
-#  - provider: script
-#    script:
-#      - ~/miniconda/bin/conda install --name root anaconda-client && ~/miniconda/bin/conda build $EARTHIO_CHANNEL_STR --output --python $PYTHON --numpy $NUMPY conda.recipe | xargs ~/miniconda/bin/conda convert -p all -o _pkgs && find _pkgs -type f -name "*.tar.bz2" -exec ~/miniconda/bin/anaconda --token $ANACONDA_UPLOAD_TOKEN upload --user $ANACONDA_UPLOAD_USER --label dev --force {} \+
-#    on:
-#      tags: false
-#      all_branches: true
-#    skip_cleanup: true
+deploy:
+  - provider: script
+    script:
+      - ~/miniconda/bin/conda install --name root anaconda-client && ~/miniconda/bin/conda build $INSTALL_CHANNELS --output --python $PYTHON --numpy $NUMPY conda.recipe | xargs ~/miniconda/bin/conda convert -p all -o _pkgs && find _pkgs -type f -name "*.tar.bz2" -exec ~/miniconda/bin/anaconda --token $ANACONDA_UPLOAD_TOKEN upload --user $ANACONDA_UPLOAD_USER --label dev --force {} \+
+    on:
+      tags: false
+      all_branches: true
+      skip_cleanup: true

MANIFEST.in

@@ -1,2 +1,3 @@
 include elm/config/defaults/environment_vars_spec.yaml
 include elm/config/defaults/config_standard.yaml
+include elm/tests/test_config.yaml

build_elm_env.sh

@@ -3,51 +3,29 @@
 set -e
 
 export ELM_BUILD_DIR=`pwd -P`
-export EARTHIO_VERSION="${EARTHIO_VERSION:-master}"
-
-if [ \( "$EARTHIO_INSTALL_METHOD" = "conda" \) -o \( "$EARTHIO_INSTALL_METHOD" = "git" \) ]; then
-    rm -rf .earthio_tmp
-    git clone http://github.com/ContinuumIO/earthio .earthio_tmp
-    cd .earthio_tmp
-    git fetch --all
-    echo git checkout $EARTHIO_VERSION
-    git checkout $EARTHIO_VERSION
-
-    set +e
-    IGNORE_ELM_DATA_DOWNLOAD=1 . build_earthio_env.sh
-    set -e
-else
-    if [ ! -d "$HOME/miniconda" ]; then
-        wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
-        bash miniconda.sh -b -p $HOME/miniconda
-    fi
+
+if [ ! -d "$HOME/miniconda" ]; then
+    wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
+    bash miniconda.sh -b -p $HOME/miniconda
     export PATH="$HOME/miniconda/bin:$PATH"
     source deactivate
-    conda config --set always_yes true
-    conda config --set anaconda_upload no
-    conda install -n root conda conda-build
-
-    # Create $EARTHIO_TEST_ENV
-    conda env remove -n $EARTHIO_TEST_ENV || true
-    conda create -n $EARTHIO_TEST_ENV $EARTHIO_CHANNEL_STR -c elm -y python=$PYTHON numpy=$NUMPY earthio
-
-    # Add earthio package to index
-    mkdir -p ~/miniconda/conda-bld/linux-64/
-    cp -av ~/miniconda/pkgs/earthio*.tar.bz2 ~/miniconda/conda-bld/linux-64/
-    cd ~/miniconda/conda-bld
-    conda index
-    cd -
+else
+    source deactivate
+    export PATH="$PATH:$(dirname $(which python))"
 fi
 
-conda remove -n root elm &> /dev/null || true
-pip uninstall -y elm &> /dev/null || true
+conda config --set always_yes true
+conda config --set anaconda_upload no
+conda install -n root conda conda-build
+
+# Create $TEST_ENV
+conda env remove -n $TEST_ENV || true
 
 cd $ELM_BUILD_DIR
 
-conda build $EARTHIO_CHANNEL_STR --python $PYTHON --numpy $NUMPY conda.recipe
-conda install -n $EARTHIO_TEST_ENV $EARTHIO_CHANNEL_STR --use-local python=$PYTHON numpy=$NUMPY elm
-for repo in "dask-glm" "dask-searchcv";do
-    # TODO improve with packaging later for ^^ dask packages
-    git clone "https://github.com/dask/${repo}" && cd $repo && python setup.py install;
-done
+conda remove -n root elm &> /dev/null || true
+pip uninstall -y elm &> /dev/null || true
+
+conda build $INSTALL_CHANNELS --python $PYTHON --numpy $NUMPY conda.recipe
+conda create -n $TEST_ENV $INSTALL_CHANNELS --use-local python=$PYTHON numpy=$NUMPY elm
 set +e

conda.recipe/meta.yaml

@@ -11,20 +11,20 @@ build:
 requirements:
   build:
     - python
+    - numpy
     - setuptools
 
   run:
-    - attrs
     - deap
+    - dask
+    - dask-searchcv
     - dill
     - distributed
-    - earthio
     - networkx
     - numba
     - numpy
     - pandas
     - python
-    - requests
     - scikit-image
     - scikit-learn
     - scipy
@@ -46,7 +46,7 @@ test:
   imports:
     - elm.config
     - elm.mldataset
-    - elm.model_selection
+    #- elm.model_selection
     - elm.pipeline.pipeline
     - elm.pipeline.steps
     - elm.scripts

elm/config/cli.py

@@ -1,4 +1,4 @@
-from __future__ import absolute_import, division, print_function, unicode_literals
+from __future__ import absolute_import, division, print_function
 
 '''Module of helpers for building command line interfaces'''
 from argparse import ArgumentParser

elm/config/config_info.py

@@ -1,4 +1,4 @@
-from __future__ import absolute_import, division, print_function, unicode_literals
+from __future__ import absolute_import, division, print_function
 
 '''
 This module loads elm/config/defaults/config_standard.yaml which

elm/config/dask_settings.py

@@ -1,4 +1,4 @@
-from __future__ import absolute_import, division, print_function, unicode_literals
+from __future__ import absolute_import, division, print_function
 
 '''
 dask_settings.py is a module of helpers for dask executors

elm/config/env.py

@@ -1,4 +1,4 @@
-from __future__ import absolute_import, division, print_function, unicode_literals
+from __future__ import absolute_import, division, print_function
 
 '''This module parses environment variables used by elm.
 

elm/config/load_config.py

@@ -1,4 +1,4 @@
-from __future__ import absolute_import, division, print_function, unicode_literals
+from __future__ import absolute_import, division, print_function
 
 '''
 This module is used by the command line interface of elm

elm/config/logging_config.py

@@ -1,4 +1,4 @@
-from __future__ import absolute_import, division, print_function, unicode_literals
+from __future__ import absolute_import, division, print_function
 
 import logging
 import os

elm/config/tests/fixtures.py

@@ -1,4 +1,4 @@
-from __future__ import absolute_import, division, print_function, unicode_literals
+from __future__ import absolute_import, division, print_function
 
 import os
 

elm/config/tests/test_config_simple.py

@@ -1,4 +1,4 @@
-from __future__ import absolute_import, division, print_function, unicode_literals
+from __future__ import absolute_import, division, print_function
 
 import copy
 import os
@@ -51,7 +51,7 @@ def tst_bad_config(bad_config):
     return ok_config
 
 def test_bad_train_config():
-
+    pytest.skip('Deprecated (temporarily) elm.config')
     bad_config = copy.deepcopy(DEFAULTS)
     name = tuple(bad_config['train'].keys())[0]
     for item in NOT_DICT + (None,):
@@ -82,6 +82,7 @@ def test_bad_train_config():
 
 
 def test_bad_pipeline():
+    pytest.skip('Deprecated (temporarily) elm.config')
     bad_config = copy.deepcopy(DEFAULTS)
     for item in NOT_LIST:
         bad_config['run'] = item

elm/config/util.py

@@ -1,4 +1,4 @@
-from __future__ import absolute_import, division, print_function, unicode_literals
+from __future__ import absolute_import, division, print_function
 
 
 from pkg_resources import resource_stream, Requirement, resource_filename

elm/mldataset/__init__.py

@@ -0,0 +1 @@
+from elm.mldataset.util import is_mldataset

elm/mldataset/serialize_mixin.py

@@ -1,5 +1,6 @@
-from __future__ import (absolute_import, division, print_function, unicode_literals,)
+from __future__ import (absolute_import, division, print_function,)
 import dill
+
 class SerializeMixin:
     '''A mixin for serialization of estimators via dill'''
     def dumps(self, protocol=None, byref=None, fmode=None, recurse=None):

elm/mldataset/util.py

@@ -0,0 +1,45 @@
+import numpy as np
+import dask.array as da
+
+from collections import Sequence
+
+
+def is_mldataset(arr, raise_err=False):
+    try:
+        from xarray_filters import MLDataset
+        from xarray import Dataset
+    except Exception as e:
+        MLDataset = Dataset = None
+        if not raise_err:
+            return False
+        # Much of the ML logic
+        # wrapping Xarray would fail
+        # if only xarray and not Xarray_filters
+        # is installed, but when xarray_filters
+        # is installed, xarray.Dataset can be
+        # used
+        raise ValueError('Cannot use cross validation for xarray Dataset without xarray_filters')
+    return MLDataset and Dataset and isinstance(arr, (MLDataset, Dataset))
+
+
+def is_arr(arr, raise_err=False):
+    is_ml = is_mldataset(arr, raise_err=raise_err)
+    _is_arr = is_ml or isinstance(arr, (np.ndarray, da.Array))
+    if not _is_arr and raise_err:
+        raise ValueError('Expected MLDataset, Dataset or Dask/Numpy array')
+    return _is_arr
+
+
+def _is_xy_tuple(result, typ=tuple):
+    if typ and not isinstance(typ, tuple):
+        typ = (typ,)
+    typ = typ + (tuple,)
+    return isinstance(result, typ) and len(result) == 2
+
+
+def _split_transformer_result(X, y, typ=tuple):
+    if _is_xy_tuple(X, typ=typ):
+        X, y2 = X
+        if y2 is not None and y is None:
+            y = y2
+    return X, y