Merge pull request #255 from paxtonfitzpatrick/master

updates to example data, DataGeometry persistence, tests, requirements + other improvements

Author: paxtonfitzpatrick

.travis.yml

@@ -4,25 +4,25 @@ python:
 - '3.6'
 - '3.7'
 - '3.8'
+- '3.9'
 services:
   - xvfb
 install:
-- wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh
-- bash miniconda.sh -b -p $HOME/miniconda
-- export PATH="$HOME/miniconda/bin:$PATH"
+- wget https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-Linux-$(uname -m).sh -O mambaforge.sh
+- bash mambaforge.sh -b -p $HOME/mambaforge
+- export PATH="$HOME/mambaforge/bin:$PATH"
 - hash -r
 - conda config --set always_yes yes --set changeps1 no
-- conda update -q conda
-- conda info -a
-- conda create -q -n testenv python=$TRAVIS_PYTHON_VERSION pip pytest numpy pandas
-  scipy matplotlib seaborn scikit-learn numba
+- mamba info -a
+- mamba create -y -q -n testenv python=$TRAVIS_PYTHON_VERSION pip pytest numpy pandas
+  scipy matplotlib seaborn scikit-learn umap-learn requests
 - source activate testenv
-- pip install -r requirements.txt
-- python setup.py install
+- pip install .
 - cp tests/matplotlibrc .
 before_script:
+- if [[ -d "$HOME/hypertools_data" ]]; then rm -rf "$HOME/hypertools_data"; fi
 - if [ ${TRAVIS_PYTHON_VERSION} == '3.6' ]; then pip install flake8; flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics; fi
-script: py.test
+script: pytest -sv .
 notifications:
   slack:
     rooms:

hypertools/_externals/ppca.py

@@ -1,7 +1,4 @@
-from __future__ import division
-from __future__ import print_function
 
-from builtins import object
 import os
 
 import numpy as np

hypertools/_externals/srm.py

@@ -33,8 +33,6 @@
 
 # Authors: Po-Hsuan Chen (Princeton Neuroscience Institute) and Javier Turek
 # (Intel Labs), 2015
-from __future__ import division
-
 import logging
 
 import numpy as np

hypertools/_shared/exceptions.py

@@ -6,3 +6,9 @@ class HypertoolsBackendError(HypertoolsError):
     def __init__(self, message):
         super().__init__(message)
         self.message = message
+
+
+class HypertoolsIOError(HypertoolsError, OSError):
+    def __init__(self, message):
+        super().__init__(message)
+        self.message = message

hypertools/_shared/helpers.py

@@ -5,12 +5,9 @@
 """
 
 ##PACKAGES##
-from __future__ import division
-from __future__ import print_function
 import functools
 import sys
 import numpy as np
-import six
 import copy
 from scipy.interpolate import PchipInterpolator as pchip
 import seaborn as sns
@@ -168,11 +165,10 @@ def get_type(data):
     """
     Checks what the data type is and returns it as a string label
     """
-    import six
     from ..datageometry import DataGeometry
 
     if isinstance(data, list):
-        if isinstance(data[0], (six.string_types, six.text_type, six.binary_type)):
+        if isinstance(data[0], (str, bytes)):
             return 'list_str'
         elif isinstance(data[0], (int, float)):
             return 'list_num'
@@ -183,13 +179,13 @@ def get_type(data):
                             'Numpy Array, Pandas DataFrame, String, List of strings'
                             ', List of numbers')
     elif isinstance(data, np.ndarray):
-        if isinstance(data[0][0], (six.string_types, six.text_type, six.binary_type)):
+        if isinstance(data[0][0], (str, bytes)):
             return 'arr_str'
         else:
             return 'arr_num'
     elif isinstance(data, pd.DataFrame):
         return 'df'
-    elif isinstance(data, (six.string_types, six.text_type, six.binary_type)):
+    elif isinstance(data, (str, bytes)):
         return 'str'
     elif isinstance(data, DataGeometry):
         return 'geo'
@@ -211,19 +207,19 @@ def check_geo(geo):
     geo = copy.copy(geo)
 
     def fix_item(item):
-        if isinstance(item, six.binary_type):
+        if isinstance(item, bytes):
             return item.decode()
         return item
 
     def fix_list(lst):
         return [fix_item(i) for i in lst]
-    if isinstance(geo.reduce, six.binary_type):
+    if isinstance(geo.reduce, bytes):
         geo.reduce = geo.reduce.decode()
     for key in geo.kwargs.keys():
         if geo.kwargs[key] is not None:
             if isinstance(geo.kwargs[key], (list, np.ndarray)):
                 geo.kwargs[key] = fix_list(geo.kwargs[key])
-            elif isinstance(geo.kwargs[key], six.binary_type):
+            elif isinstance(geo.kwargs[key], bytes):
                 geo.kwargs[key] = fix_item(geo.kwargs[key])
     return geo
 
@@ -232,7 +228,6 @@ def get_dtype(data):
     """
     Checks what the data type is and returns it as a string label
     """
-    import six
     from ..datageometry import DataGeometry
 
     if isinstance(data, list):
@@ -241,7 +236,7 @@ def get_dtype(data):
         return 'arr'
     elif isinstance(data, pd.DataFrame):
         return 'df'
-    elif isinstance(data, (six.string_types, six.text_type, six.binary_type)):
+    elif isinstance(data, (str, bytes)):
         return 'str'
     elif isinstance(data, DataGeometry):
         return 'geo'

hypertools/datageometry.py

@@ -1,8 +1,9 @@
-from __future__ import unicode_literals
-from builtins import object
 import copy
-import deepdish as dd
-import numpy as np
+import pickle
+import warnings
+
+import pandas as pd
+
 from .tools.normalize import normalize as normalizer
 from .tools.reduce import reduce as reducer
 from .tools.align import align as aligner
@@ -188,7 +189,7 @@ def plot(self, data=None, **kwargs):
             new_kwargs.update({key : kwargs[key]})
         return plotter(d, **new_kwargs)
 
-    def save(self, fname, compression='blosc'):
+    def save(self, fname, compression=None):
         """
         Save method for the data geometry object
 
@@ -198,41 +199,46 @@ def save(self, fname, compression='blosc'):
 
         Parameters
         ----------
-
         fname : str
             A name for the file.  If the file extension (.geo) is not specified,
             it will be appended.
-
-        compression : str
-            The kind of compression to use.  See the deepdish documentation for
-            options: http://deepdish.readthedocs.io/en/latest/api_io.html#deepdish.io.save
-
         """
-        if hasattr(self, 'dtype'):
-            if 'list' in self.dtype:
-                data = np.array(self.data)
-            elif 'df' in self.dtype:
-                data = {k: np.array(v).astype('str') for k, v in self.data.to_dict('list').items()}
-            else:
-                data = self.data
-
-        # put geo vars into a dict
-        geo = {
-            'data' : data,
-            'xform_data' : np.array(self.xform_data),
-            'reduce' : self.reduce,
-            'align' : self.align,
-            'normalize' : self.normalize,
-            'semantic' : self.semantic,
-            'corpus' : np.array(self.corpus) if isinstance(self.corpus, list) else self.corpus,
-            'kwargs' : self.kwargs,
-            'version' : self.version,
-            'dtype' : self.dtype
-        }
-
-        # if extension wasn't included, add it
-        if fname[-4:]!='.geo':
-            fname+='.geo'
-
-        # save
-        dd.io.save(fname, geo, compression=compression)
+        if compression is not None:
+            warnings.warn("Hypertools has switched from deepdish to pickle "
+                          "for saving DataGeomtry objects. 'compression' "
+                          "argument has no effect and will be removed in a "
+                          "future version",
+                          FutureWarning)
+
+        # automatically add extension if not present
+        if not fname.endswith('.geo'):
+            fname += '.geo'
+
+        # can't save/restore matplotlib objects across sessions
+        curr_fig = self.fig
+        curr_ax = self.ax
+        curr_line_ani = self.line_ani
+
+        curr_data = self.data
+        # convert pandas DataFrames to dicts of
+        # {column_name: list(column_values)} to fix I/O compatibility
+        # issues across certain pandas versions. Expected self.data
+        # format is restored by hypertools.load
+        if isinstance(curr_data, pd.DataFrame):
+            data_out_fmt = curr_data.to_dict('list')
+        else:
+            data_out_fmt = curr_data
+
+        try:
+            self.fig = self.ax = self.line_ani = None
+            self.data = data_out_fmt
+            # save
+            with open(fname, 'wb') as f:
+                pickle.dump(self, f)
+        finally:
+            # make sure we don't mutate attribute values whether or not
+            # save was successful
+            self.fig = curr_fig
+            self.ax = curr_ax
+            self.line_ani = curr_line_ani
+            self.data = curr_data

hypertools/plot/draw.py

@@ -1,8 +1,5 @@
 #!/usr/bin/env python
 
-from __future__ import division
-from builtins import str
-from builtins import range
 import matplotlib
 import matplotlib.pyplot as plt
 from mpl_toolkits.mplot3d import proj3d
@@ -330,7 +327,7 @@ def update_lines_spin(num, data_lines, lines, cube_scale, rotations=2,
         return lines
 
     def dispatch_animate(x, ani_params):
-        if x[0].shape[1] is 3:
+        if x[0].shape[1] == 3:
             return animate_plot3D(x, **ani_params)
 
     def animate_plot3D(x, tail_duration=2, rotations=2, zoom=1, chemtrails=False,
@@ -380,7 +377,7 @@ def animate_plot3D(x, tail_duration=2, rotations=2, zoom=1, chemtrails=False,
         plt.ioff()
 
     if animate in [True, 'parallel', 'spin']:
-        assert x[0].shape[1] is 3, "Animations are currently only supported for 3d plots."
+        assert x[0].shape[1] == 3, "Animations are currently only supported for 3d plots."
 
         # animation params
         ani_params = dict(tail_duration=tail_duration,
@@ -400,7 +397,7 @@ def animate_plot3D(x, tail_duration=2, rotations=2, zoom=1, chemtrails=False,
         fig, ax, data = dispatch_static(x, ax)
 
         # if 3d, plot the cube
-        if x[0].shape[1] is 3:
+        if x[0].shape[1] == 3:
 
             # set cube scale
             cube_scale = 1
@@ -416,7 +413,7 @@ def animate_plot3D(x, tail_duration=2, rotations=2, zoom=1, chemtrails=False,
             # initialize the view
             ax.view_init(elev=elev, azim=azim)
 
-        elif x[0].shape[1] is 2:
+        elif x[0].shape[1] == 2:
 
             # plot square
             plot_square(ax)

hypertools/plot/plot.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-from __future__ import division
 import warnings
 import matplotlib.animation as animation
 import matplotlib.pyplot as plt
@@ -311,7 +310,7 @@ def plot(x, fmt='-', marker=None, markers=None, linestyle=None, linestyles=None,
     if cluster is not None:
         if hue is not None:
             warnings.warn('cluster overrides hue, ignoring hue.')
-        if isinstance(cluster, (six.string_types, six.binary_type)):
+        if isinstance(cluster, (str, bytes)):
             model = cluster
             params = default_params(model)
         elif isinstance(cluster, dict):
@@ -374,7 +373,7 @@ def plot(x, fmt='-', marker=None, markers=None, linestyle=None, linestyles=None,
         mpl_kwargs['label'] = legend
 
     # interpolate if its a line plot
-    if fmt is None or isinstance(fmt, six.string_types):
+    if fmt is None or isinstance(fmt, str):
         if is_line(fmt):
             if xform[0].shape[0] > 1:
                 xform = interp_array_list(xform, interp_val=frame_rate*duration/(xform[0].shape[0] - 1))
@@ -386,8 +385,8 @@ def plot(x, fmt='-', marker=None, markers=None, linestyle=None, linestyles=None,
 
     # handle explore flag
     if explore:
-        assert xform[0].shape[1] is 3, "Explore mode is currently only supported for 3D plots."
-        mpl_kwargs['picker']=True
+        assert xform[0].shape[1] == 3, "Explore mode is currently only supported for 3D plots."
+        mpl_kwargs['picker'] = True
 
     # center
     xform = center(xform)

hypertools/tools/align.py

@@ -1,7 +1,5 @@
 #!/usr/bin/env python
 
-from __future__ import division
-from builtins import range
 from .._externals.srm import SRM
 from .procrustes import procrustes
 import numpy as np
@@ -82,7 +80,7 @@ def align(data, align='hyper', normalize=None, ndims=None, method=None,
         if format_data:
             data = formatter(data, ppca=True)
 
-        if len(data) is 1:
+        if len(data) == 1:
             warnings.warn('Data in list of length 1 can not be aligned. '
                  'Skipping the alignment.')
 

hypertools/tools/cluster.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python
 import warnings
 from sklearn.cluster import KMeans, MiniBatchKMeans, AgglomerativeClustering, Birch, FeatureAgglomeration, SpectralClustering
-import six
 import numpy as np
 from .._shared.helpers import *
 from .format_data import format_data as formatter
@@ -64,7 +63,7 @@ def cluster(x, cluster='KMeans', n_clusters=3, ndims=None, format_data=True):
 
     if cluster == None:
         return x
-    elif (isinstance(cluster, six.string_types) and cluster=='HDBSCAN') or \
+    elif (isinstance(cluster, str) and cluster=='HDBSCAN') or \
     (isinstance(cluster, dict) and cluster['model']=='HDBSCAN'):
         if not _has_hdbscan:
             raise ImportError('HDBSCAN is not installed. Please install hdbscan>=0.8.11')
@@ -76,7 +75,7 @@ def cluster(x, cluster='KMeans', n_clusters=3, ndims=None, format_data=True):
         x = formatter(x, ppca=True)
 
     # if reduce is a string, find the corresponding model
-    if isinstance(cluster, six.string_types):
+    if isinstance(cluster, str):
         model = models[cluster]
         if cluster != 'HDBSCAN':
             model_params = {
@@ -86,7 +85,7 @@ def cluster(x, cluster='KMeans', n_clusters=3, ndims=None, format_data=True):
             model_params = {}
     # if its a dict, use custom params
     elif type(cluster) is dict:
-        if isinstance(cluster['model'], six.string_types):
+        if isinstance(cluster['model'], str):
             model = models[cluster['model']]
             model_params = cluster['params']