Pandas datetime and numpy numeric array fixes (#1163)

* Fix error when setting compound or compound array property to `None`

* Pandas datetime and numpy numeric array fixes

 1) Preserve numeric numpy types as is in validator out, even if that
    numeric type is not supported as JavaScript TypedArray
 2) Update widget serializer to check numeric numpy arrays for whether
    they are compatible with TypedArrays. If not, serialize as list.
 3) Call to_pydatetime() on pandas datetime series/index values
    when passed to copy_to_readonly_numpy_array. This returns numpy
    array of datetimes (which we already know how to serialize)

Fixes datetime issue in #1160
Fixes FigureWidget issue in #1155

Author: jonmmease

_plotly_utils/basevalidators.py

@@ -52,17 +52,18 @@ def to_scalar_or_list(v):
         return v
 
 
-def copy_to_readonly_numpy_array(v, dtype=None, force_numeric=False):
+def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):
     """
     Convert an array-like value into a read-only numpy array
 
     Parameters
     ----------
     v : array like
         Array like value (list, tuple, numpy array, pandas series, etc.)
-    dtype : str
-        If specified, the numpy dtype that the array should be forced to
-        have. If not specified then let numpy infer the datatype
+    kind : str or tuple of str
+        If specified, the numpy dtype kind (or kinds) that the array should
+        have, or be converted to if possible.
+        If not specified then let numpy infer the datatype
     force_numeric : bool
         If true, raise an exception if the resulting numpy array does not
         have a numeric dtype (i.e. dtype.kind not in ['u', 'i', 'f'])
@@ -74,30 +75,53 @@ def copy_to_readonly_numpy_array(v, dtype=None, force_numeric=False):
 
     assert np is not None
 
-    # Copy to numpy array and handle dtype param
-    # ------------------------------------------
-    # If dtype was not specified then it will be passed to the numpy array
-    # constructor as None and the data type will be inferred automatically
+    # ### Process kind ###
+    if not kind:
+        kind = ()
+    elif isinstance(kind, string_types):
+        kind = (kind,)
+
+    first_kind = kind[0] if kind else None
 
-    # TODO: support datetime dtype here and in widget serialization
     # u: unsigned int, i: signed int, f: float
-    numeric_kinds = ['u', 'i', 'f']
+    numeric_kinds = {'u', 'i', 'f'}
+    kind_default_dtypes = {
+        'u': 'uint32', 'i': 'int32', 'f': 'float64', 'O': 'object'}
 
-    # Unwrap data types that have a `values` property that might be a numpy
-    # array. If this values property is a numeric numpy array then we
-    # can take the fast path below
+    # Handle pandas Series and Index objects
     if pd and isinstance(v, (pd.Series, pd.Index)):
-        v = v.values
+        if v.dtype.kind in numeric_kinds:
+            # Get the numeric numpy array so we use fast path below
+            v = v.values
+        elif v.dtype.kind == 'M':
+            # Convert datetime Series/Index to numpy array of datetimes
+            if isinstance(v, pd.Series):
+                v = v.dt.to_pydatetime()
+            else:
+                # DatetimeIndex
+                v = v.to_pydatetime()
 
     if not isinstance(v, np.ndarray):
+        # v is not homogenous array
         v_list = [to_scalar_or_list(e) for e in v]
+
+        # Lookup dtype for requested kind, if any
+        dtype = kind_default_dtypes.get(first_kind, None)
+
+        # construct new array from list
         new_v = np.array(v_list, order='C', dtype=dtype)
     elif v.dtype.kind in numeric_kinds:
-        if dtype:
+        # v is a homogenous numeric array
+        if kind and v.dtype.kind not in kind:
+            # Kind(s) were specified and this array doesn't match
+            # Convert to the default dtype for the first kind
+            dtype = kind_default_dtypes.get(first_kind, None)
             new_v = np.ascontiguousarray(v.astype(dtype))
         else:
+            # Either no kind was requested or requested kind is satisfied
             new_v = np.ascontiguousarray(v.copy())
     else:
+        # v is a non-numeric homogenous array
         new_v = v.copy()
 
     # Handle force numeric param
@@ -106,7 +130,7 @@ def copy_to_readonly_numpy_array(v, dtype=None, force_numeric=False):
         raise ValueError('Input value is not numeric and'
                          'force_numeric parameter set to True')
 
-    if dtype != 'unicode':
+    if 'U' not in kind:
         # Force non-numeric arrays to have object type
         # --------------------------------------------
         # Here we make sure that non-numeric arrays have the object
@@ -116,12 +140,6 @@ def copy_to_readonly_numpy_array(v, dtype=None, force_numeric=False):
         if new_v.dtype.kind not in ['u', 'i', 'f', 'O']:
             new_v = np.array(v, dtype='object')
 
-    # Convert int64 arrays to int32
-    # -----------------------------
-    # JavaScript doesn't support int64 typed arrays
-    if new_v.dtype == 'int64':
-        new_v = new_v.astype('int32')
-
     # Set new array to be read-only
     # -----------------------------
     new_v.flags['WRITEABLE'] = False
@@ -749,10 +767,13 @@ def validate_coerce(self, v):
             # Pass None through
             pass
         elif self.array_ok and is_homogeneous_array(v):
-            if v.dtype.kind not in ['i', 'u']:
-                self.raise_invalid_val(v)
 
-            v_array = copy_to_readonly_numpy_array(v, dtype='int32')
+            v_array = copy_to_readonly_numpy_array(v,
+                                                   kind=('i', 'u'),
+                                                   force_numeric=True)
+
+            if v_array.dtype.kind not in ['i', 'u']:
+                self.raise_invalid_val(v)
 
             # Check min/max
             if self.has_min_max:
@@ -875,7 +896,7 @@ def validate_coerce(self, v):
 
             if is_homogeneous_array(v):
                 # If not strict, let numpy cast elements to strings
-                v = copy_to_readonly_numpy_array(v, dtype='unicode')
+                v = copy_to_readonly_numpy_array(v, kind='U')
 
                 # Check no_blank
                 if self.no_blank:
@@ -1057,10 +1078,10 @@ def validate_coerce(self, v, should_raise=True):
                 # ### Check that elements have valid colors types ###
                 elif self.numbers_allowed() or invalid_els:
                     v = copy_to_readonly_numpy_array(
-                        validated_v, dtype='object')
+                        validated_v, kind='O')
                 else:
                     v = copy_to_readonly_numpy_array(
-                        validated_v, dtype='unicode')
+                        validated_v, kind='U')
         elif self.array_ok and is_simple_array(v):
             validated_v = [
                 self.validate_coerce(e, should_raise=False)
@@ -1509,7 +1530,7 @@ def validate_coerce(self, v):
                 self.raise_invalid_elements(invalid_els)
 
             if is_homogeneous_array(v):
-                v = copy_to_readonly_numpy_array(validated_v, dtype='unicode')
+                v = copy_to_readonly_numpy_array(validated_v, kind='U')
             else:
                 v = to_scalar_or_list(v)
         else:
@@ -1559,7 +1580,7 @@ def validate_coerce(self, v):
             # Pass None through
             pass
         elif self.array_ok and is_homogeneous_array(v):
-            v = copy_to_readonly_numpy_array(v, dtype='object')
+            v = copy_to_readonly_numpy_array(v, kind='O')
         elif self.array_ok and is_simple_array(v):
             v = to_scalar_or_list(v)
         return v

_plotly_utils/tests/validators/test_integer_validator.py

@@ -128,7 +128,7 @@ def test_acceptance_aok_list(val, validator_aok):
 def test_coercion_aok_list(val, expected, validator_aok):
     v = validator_aok.validate_coerce(val)
     if isinstance(val, (np.ndarray, pd.Series, pd.Index)):
-        assert v.dtype == np.int32
+        assert v.dtype == val.dtype
         assert np.array_equal(validator_aok.present(v),
                               np.array(expected, dtype=np.int32))
     else:

_plotly_utils/tests/validators/test_pandas_series_input.py

@@ -0,0 +1,179 @@
+import pytest
+import numpy as np
+import pandas as pd
+from datetime import datetime
+from _plotly_utils.basevalidators import (NumberValidator,
+                                          IntegerValidator,
+                                          DataArrayValidator,
+                                          ColorValidator)
+
+
+@pytest.fixture
+def data_array_validator(request):
+    return DataArrayValidator('prop', 'parent')
+
+
+@pytest.fixture
+def integer_validator(request):
+    return IntegerValidator('prop', 'parent', array_ok=True)
+
+
+@pytest.fixture
+def number_validator(request):
+    return NumberValidator('prop', 'parent', array_ok=True)
+
+
+@pytest.fixture
+def color_validator(request):
+    return ColorValidator('prop', 'parent', array_ok=True, colorscale_path='')
+
+
+@pytest.fixture(
+    params=['int8', 'int16', 'int32', 'int64',
+            'uint8', 'uint16', 'uint32', 'uint64',
+            'float16', 'float32', 'float64'])
+def numeric_dtype(request):
+    return request.param
+
+
+@pytest.fixture(
+    params=[pd.Series, pd.Index])
+def pandas_type(request):
+    return request.param
+
+
+@pytest.fixture
+def numeric_pandas(request, pandas_type, numeric_dtype):
+    return pandas_type(np.arange(10), dtype=numeric_dtype)
+
+
+@pytest.fixture
+def color_object_pandas(request, pandas_type):
+    return pandas_type(['blue', 'green', 'red']*3, dtype='object')
+
+
+@pytest.fixture
+def color_categorical_pandas(request, pandas_type):
+    return pandas_type(pd.Categorical(['blue', 'green', 'red']*3))
+
+
+@pytest.fixture
+def dates_array(request):
+    return np.array([
+        datetime(year=2013, month=10, day=10),
+        datetime(year=2013, month=11, day=10),
+        datetime(year=2013, month=12, day=10),
+        datetime(year=2014, month=1, day=10),
+        datetime(year=2014, month=2, day=10)
+    ])
+
+
+@pytest.fixture
+def datetime_pandas(request, pandas_type, dates_array):
+    return pandas_type(dates_array)
+
+
+def test_numeric_validator_numeric_pandas(number_validator, numeric_pandas):
+    res = number_validator.validate_coerce(numeric_pandas)
+
+    # Check type
+    assert isinstance(res, np.ndarray)
+
+    # Check dtype
+    assert res.dtype == numeric_pandas.dtype
+
+    # Check values
+    np.testing.assert_array_equal(res, numeric_pandas)
+
+
+def test_integer_validator_numeric_pandas(integer_validator, numeric_pandas):
+    res = integer_validator.validate_coerce(numeric_pandas)
+
+    # Check type
+    assert isinstance(res, np.ndarray)
+
+    # Check dtype
+    if numeric_pandas.dtype.kind in ('u', 'i'):
+        # Integer and unsigned integer dtype unchanged
+        assert res.dtype == numeric_pandas.dtype
+    else:
+        # Float datatypes converted to default integer type of int32
+        assert res.dtype == 'int32'
+
+    # Check values
+    np.testing.assert_array_equal(res, numeric_pandas)
+
+
+def test_data_array_validator(data_array_validator,
+                              numeric_pandas):
+    res = data_array_validator.validate_coerce(numeric_pandas)
+
+    # Check type
+    assert isinstance(res, np.ndarray)
+
+    # Check dtype
+    assert res.dtype == numeric_pandas.dtype
+
+    # Check values
+    np.testing.assert_array_equal(res, numeric_pandas)
+
+
+def test_color_validator_numeric(color_validator,
+                                 numeric_pandas):
+    res = color_validator.validate_coerce(numeric_pandas)
+
+    # Check type
+    assert isinstance(res, np.ndarray)
+
+    # Check dtype
+    assert res.dtype == numeric_pandas.dtype
+
+    # Check values
+    np.testing.assert_array_equal(res, numeric_pandas)
+
+
+def test_color_validator_object(color_validator,
+                                color_object_pandas):
+
+    res = color_validator.validate_coerce(color_object_pandas)
+
+    # Check type
+    assert isinstance(res, np.ndarray)
+
+    # Check dtype
+    assert res.dtype == 'object'
+
+    # Check values
+    np.testing.assert_array_equal(res, color_object_pandas)
+
+
+def test_color_validator_categorical(color_validator,
+                                     color_categorical_pandas):
+
+    res = color_validator.validate_coerce(color_categorical_pandas)
+
+    # Check type
+    assert color_categorical_pandas.dtype == 'category'
+    assert isinstance(res, np.ndarray)
+
+    # Check dtype
+    assert res.dtype == 'object'
+
+    # Check values
+    np.testing.assert_array_equal(res, np.array(color_categorical_pandas))
+
+
+def test_data_array_validator_dates(data_array_validator,
+                                    datetime_pandas,
+                                    dates_array):
+
+    res = data_array_validator.validate_coerce(datetime_pandas)
+
+    # Check type
+    assert isinstance(res, np.ndarray)
+
+    # Check dtype
+    assert res.dtype == 'object'
+
+    # Check values
+    np.testing.assert_array_equal(res, dates_array)

plotly/basedatatypes.py

@@ -2981,7 +2981,7 @@ def _set_compound_prop(self, prop, val):
         # ------------------
         if not self._in_batch_mode:
             if not new_dict_val:
-                if prop in self._props:
+                if self._props and prop in self._props:
                     self._props.pop(prop)
             else:
                 self._init_props()
@@ -3055,7 +3055,7 @@ def _set_array_prop(self, prop, val):
         # ------------------
         if not self._in_batch_mode:
             if not new_dict_vals:
-                if prop in self._props:
+                if self._props and prop in self._props:
                     self._props.pop(prop)
             else:
                 self._init_props()