Enhancement: configuration of serialization for single element strings

Author: maciejlach

CHANGELOG.txt

@@ -1,8 +1,14 @@
+------------------------------------------------------------------------------
+  qPython 1.0.2 [2015.09.03]
+------------------------------------------------------------------------------
+
+  - Enhancement: configuration of serialization for single element strings
+
 ------------------------------------------------------------------------------
   qPython 1.0.1 [2015.08.11]
 ------------------------------------------------------------------------------
 
-  - Fix: serialization of indexed Pandas.DataFrames 
+  - Fix: serialization of indexed Pandas.DataFrames
 
 ------------------------------------------------------------------------------
   qPython 1.0.0 [2015.04.10]

conda.recipe/meta.yaml

@@ -1,6 +1,6 @@
 package:
   name: qpython
-  version: 1.0.1
+  version: 1.0.2
 
 build:
     number: 1

doc/source/conf.py

@@ -70,10 +70,11 @@ def __getattr__(cls, name):
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
+from qpython import __version__
 # The short X.Y version.
-version = '1.0'
+version = __version__
 # The full version, including alpha/beta/rc tags.
-release = '1.0.1'
+release = __version__
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -261,7 +262,7 @@ def __getattr__(cls, name):
 #  dir menu entry, description, category)
 texinfo_documents = [
   ('index', 'qPython', u'qPython Documentation',
-   u'DEVnet', 'qPython', 'One line description of project.',
+   u'DEVnet', 'qPython', 'Interprocess communication between Python and kdb+',
    'Miscellaneous'),
 ]
 

doc/source/connection.rst

@@ -30,8 +30,8 @@ can be used with a ``with`` statement:
       print q('{`int$ til x}', 10)
 
 
-Q parser configuration
-**********************
+Types conversion configuration
+******************************
 
 Connection can be preconfigured to parse IPC messages according to a specified
 settings, e.g.: temporal vectors can be represented as raw vectors or converted
@@ -45,6 +45,7 @@ to numpy `datetime64`/`timedelta64` representation.
   q = qconnection.QConnection(host = 'localhost', port = 5000, numpy_temporals = True) 
 
 
-Parsing options can be also overwritten while executing synchronous query 
-(:meth:`~qpython.qconnection.QConnection.sync`) or retrieving data from q
+Conversion options can be also overwritten while executing 
+synchronous/asynchronous queries (:meth:`~qpython.qconnection.QConnection.sync`,
+:meth:`~qpython.qconnection.QConnection.async`) or retrieving data from q
 (:meth:`~qpython.qconnection.QConnection.receive`).

doc/source/queries.rst

@@ -106,15 +106,16 @@ QMessage: message type: 2, data size: 13, is_compressed: False, data: 10
 fa0a000000
 
 
-Q parser configuration
-**********************
+Type conversions configuration
+******************************
 
-Parsing options can be overwritten while:
+Type conversion options can be overwritten while:
 
 - executing synchronous query: :meth:`~qpython.qconnection.QConnection.sync`
+- executing asynchronous query: :meth:`~qpython.qconnection.QConnection.async`
 - retrieving data from q: :meth:`~qpython.qconnection.QConnection.receive`
 
-Both methods accepts the `options` keywords arguments::
+These methods accepts the `options` keywords arguments::
 
     >>> query = "{[x] 0Nd, `date$til x}"
     
@@ -133,4 +134,10 @@ Both methods accepts the `options` keywords arguments::
     [NaT [metadata(qtype=-14)] 2000-01-01 [metadata(qtype=-14)]
      2000-01-02 [metadata(qtype=-14)] 2000-01-03 [metadata(qtype=-14)]]
     
-    
+    >>> # serialize single element strings as q characters 
+    >>> print q.sync('{[x] type each x}', ['one', 'two', '3'], single_char_strings = False)
+    [ 10,  10, -10]
+    
+    >>> # serialize single element strings as q strings 
+    >>> print q.sync('{[x] type each x}', ['one', 'two', '3'], single_char_strings = True)
+    [10, 10, 10]

doc/source/type-conversion.rst

@@ -22,14 +22,14 @@ to this table:
 ===============  ============ =====================================
  q  type          q num type   Python type        
 ===============  ============ =====================================
- ``bool``         -1           ``numpy.bool_``        
+ ``bool``         -1           ``numpy.bool_``
  ``guid``         -2           ``UUID``
- ``byte``         -4           ``numpy.byte``         
- ``short``        -5           ``numpy.int16``        
- ``integer``      -6           ``numpy.int32``        
- ``long``         -7           ``numpy.int64``        
- ``real``         -8           ``numpy.float32``      
- ``float``        -9           ``numpy.float64``      
+ ``byte``         -4           ``numpy.byte``
+ ``short``        -5           ``numpy.int16``
+ ``integer``      -6           ``numpy.int32``
+ ``long``         -7           ``numpy.int64``
+ ``real``         -8           ``numpy.float32``
+ ``float``        -9           ``numpy.float64``
  ``character``    -10          single element ``str``
  ``timestamp``    -12          ``QTemporal  numpy.datetime64   ns``
  ``month``        -13          ``QTemporal  numpy.datetime64   M``
@@ -54,31 +54,35 @@ described in the table:
 =====================================  ================  ============
  Python type                            q type            q num type 
 =====================================  ================  ============
- ``bool``                               ``bool``          -1         
- ---                                    ``byte``          -4         
- ---                                    ``short``         -5         
- ``int``                                ``int``           -6         
- ``long``                               ``long``          -7         
- ---                                    ``real``          -8         
- ``double``                             ``float``         -9         
- ``numpy.bool``                         ``bool``          -1         
- ``numpy.byte``                         ``byte``          -4         
- ``numpy.int16``                        ``short``         -5         
- ``numpy.int32``                        ``int``           -6         
- ``numpy.int64``                        ``long``          -7         
- ``numpy.float32``                      ``real``          -8         
- ``numpy.float64``                      ``float``         -9         
- single element ``str``                 ``character``     -10        
- ``QTemporal  numpy.datetime64   ns``   ``timestamp``     -12                
- ``QTemporal  numpy.datetime64   M``    ``month``         -13                
- ``QTemporal  numpy.datetime64   D``    ``date``          -14              
- ``QTemporal  numpy.datetime64   ms``   ``datetime``      -15              
- ``QTemporal  numpy.timedelta64  ns``   ``timespan``      -16              
- ``QTemporal  numpy.timedelta64  m``    ``minute``        -17              
- ``QTemporal  numpy.timedelta64  s``    ``second``        -18              
- ``QTemporal  numpy.timedelta64  ms``   ``time``          -19              
+ ``bool``                               ``bool``          -1
+ ---                                    ``byte``          -4
+ ---                                    ``short``         -5
+ ``int``                                ``int``           -6
+ ``long``                               ``long``          -7
+ ---                                    ``real``          -8
+ ``double``                             ``float``         -9
+ ``numpy.bool``                         ``bool``          -1
+ ``numpy.byte``                         ``byte``          -4
+ ``numpy.int16``                        ``short``         -5
+ ``numpy.int32``                        ``int``           -6
+ ``numpy.int64``                        ``long``          -7
+ ``numpy.float32``                      ``real``          -8
+ ``numpy.float64``                      ``float``         -9
+ single element ``str``                 ``character``     -10
+ ``QTemporal  numpy.datetime64   ns``   ``timestamp``     -12
+ ``QTemporal  numpy.datetime64   M``    ``month``         -13
+ ``QTemporal  numpy.datetime64   D``    ``date``          -14
+ ``QTemporal  numpy.datetime64   ms``   ``datetime``      -15
+ ``QTemporal  numpy.timedelta64  ns``   ``timespan``      -16
+ ``QTemporal  numpy.timedelta64  m``    ``minute``        -17
+ ``QTemporal  numpy.timedelta64  s``    ``second``        -18
+ ``QTemporal  numpy.timedelta64  ms``   ``time``          -19
 =====================================  ================  ============
 
+.. note:: By default, single element strings are serialized as q characters. 
+          This setting can be modified (`single_char_strings = True`) and 
+          and single element strings are represented as q strings.
+
 
 String and symbols
 ******************
@@ -98,6 +102,20 @@ apply:
     'quick brown fox jumps over a lazy dog'
 
 
+.. note:: By default, single element strings are serialized as q characters. 
+          This setting can be modified (`single_char_strings = True`) and 
+          and single element strings are represented as q strings.
+
+::
+          
+    >>> # serialize single element strings as q characters 
+    >>> print(q.sync('{[x] type each x}', ['one', 'two', '3'], single_char_strings = False))
+    [ 10,  10, -10]
+    
+    >>> # serialize single element strings as q strings 
+    >>> print(q.sync('{[x] type each x}', ['one', 'two', '3'], single_char_strings = True))
+    [10, 10, 10]
+
 
 Lists
 *****
@@ -350,22 +368,22 @@ Complete null mapping between q and Python is represented in the table:
 ============== ============== =======================
  ``bool``       ``0b``          ``_QNULL_BOOL``
  ``guid``       ``0Ng``         ``_QNULL_GUID``
- ``byte``       ``0x00``        ``_QNULL1``     
- ``short``      ``0Nh``         ``_QNULL2``     
- ``int``        ``0N``          ``_QNULL4``     
- ``long``       ``0Nj``         ``_QNULL8``     
- ``real``       ``0Ne``         ``_QNAN32``     
- ``float``      ``0n``          ``_QNAN64``     
- ``string``     ``" "``         ``' '``         
+ ``byte``       ``0x00``        ``_QNULL1``
+ ``short``      ``0Nh``         ``_QNULL2``
+ ``int``        ``0N``          ``_QNULL4``
+ ``long``       ``0Nj``         ``_QNULL8``
+ ``real``       ``0Ne``         ``_QNAN32``
+ ``float``      ``0n``          ``_QNAN64``
+ ``string``     ``" "``         ``' '``
  ``symbol``     \`              ``_QNULL_SYM``
- ``timestamp``  ``0Np``         ``_QNULL8``                
- ``month``      ``0Nm``         ``_QNULL4``               
- ``date``       ``0Nd``         ``_QNULL4``                  
- ``datetime``   ``0Nz``         ``_QNAN64``                  
- ``timespan``   ``0Nn``         ``_QNULL8``                  
- ``minute``     ``0Nu``         ``_QNULL4``                  
- ``second``     ``0Nv``         ``_QNULL4``                  
- ``time``       ``0Nt``         ``_QNULL4``                  
+ ``timestamp``  ``0Np``         ``_QNULL8``
+ ``month``      ``0Nm``         ``_QNULL4``
+ ``date``       ``0Nd``         ``_QNULL4``
+ ``datetime``   ``0Nz``         ``_QNAN64``
+ ``timespan``   ``0Nn``         ``_QNULL8``
+ ``minute``     ``0Nu``         ``_QNULL4``
+ ``second``     ``0Nv``         ``_QNULL4``
+ ``time``       ``0Nt``         ``_QNULL4``
 ============== ============== =======================
 
 The :py:mod:`qtype` provides two utility functions to work with null values:

qpython/__init__.py

@@ -17,7 +17,7 @@
 __all__ = ['qconnection', 'qtype', 'qtemporal', 'qcollection']
 
 
-__version__ = '1.0.1'
+__version__ = '1.0.2'
 
 try:
     from qpython.fastutils import uncompress
@@ -56,4 +56,11 @@ def as_dict(self):
         return self.__dict__.copy()
 
     def union_dict(self, **kw):
-        return dict(self.as_dict().items() + kw.items())
+        return dict(list(self.as_dict().items()) + list(kw.items()))
+
+
+CONVERSION_OPTIONS = MetaData(raw = False,
+                              numpy_temporals = False,
+                              pandas = False,
+                              single_char_strings = False
+                             )

qpython/_pandas.py

@@ -20,7 +20,7 @@
 from collections import OrderedDict
 
 from qpython import MetaData
-from qpython.qreader import QReader, READER_CONFIGURATION, QReaderException
+from qpython.qreader import QReader, QReaderException
 from qpython.qcollection import QDictionary, qlist
 from qpython.qwriter import QWriter, QWriterException
 from qpython.qtype import *
@@ -32,10 +32,10 @@ class PandasQReader(QReader):
     parse = Mapper(QReader._reader_map)
 
     @parse(QDICTIONARY)
-    def _read_dictionary(self, qtype = QDICTIONARY, options = READER_CONFIGURATION):
-        if options.pandas:
-            keys = self._read_object(options = options)
-            values = self._read_object(options = options)
+    def _read_dictionary(self, qtype = QDICTIONARY):
+        if self._options.pandas:
+            keys = self._read_object()
+            values = self._read_object()
 
             if isinstance(keys, pandas.DataFrame):
                 if not isinstance(values, pandas.DataFrame):
@@ -58,17 +58,17 @@ def _read_dictionary(self, qtype = QDICTIONARY, options = READER_CONFIGURATION):
                 values = values if not isinstance(values, pandas.Series) else values.as_matrix()
                 return QDictionary(keys, values)
         else:
-            return QReader._read_dictionary(self, qtype = qtype, options = options)
+            return QReader._read_dictionary(self, qtype = qtype)
 
 
     @parse(QTABLE)
-    def _read_table(self, qtype = QTABLE, options = READER_CONFIGURATION):
-        if options.pandas:
+    def _read_table(self, qtype = QTABLE):
+        if self._options.pandas:
             self._buffer.skip()  # ignore attributes
             self._buffer.skip()  # ignore dict type stamp
 
-            columns = self._read_object(options = options)
-            data = self._read_object(options = options)
+            columns = self._read_object()
+            data = self._read_object()
 
             odict = OrderedDict()
             meta = MetaData(qtype = QTABLE)
@@ -91,16 +91,16 @@ def _read_table(self, qtype = QTABLE, options = READER_CONFIGURATION):
             df.meta = meta
             return df
         else:
-            return QReader._read_table(self, qtype = qtype, options = options)
+            return QReader._read_table(self, qtype = qtype)
 
 
-    def _read_list(self, qtype, options):
-        if options.pandas:
-            options.numpy_temporals = True
+    def _read_list(self, qtype):
+        if self._options.pandas:
+            self._options.numpy_temporals = True
 
-        list = QReader._read_list(self, qtype = qtype, options = options)
+        list = QReader._read_list(self, qtype = qtype)
 
-        if options.pandas:
+        if self._options.pandas:
             if -abs(qtype) not in [QMONTH, QDATE, QDATETIME, QMINUTE, QSECOND, QTIME, QTIMESTAMP, QTIMESPAN, QSYMBOL]:
                 null = QNULLMAP[-abs(qtype)][1]
                 ps = pandas.Series(data = list).replace(null, numpy.NaN)
@@ -114,9 +114,9 @@ def _read_list(self, qtype, options):
 
 
     @parse(QGENERAL_LIST)
-    def _read_general_list(self, qtype = QGENERAL_LIST, options = READER_CONFIGURATION):
-        list = QReader._read_general_list(self, qtype, options)
-        if options.pandas:
+    def _read_general_list(self, qtype = QGENERAL_LIST):
+        list = QReader._read_general_list(self, qtype)
+        if self._options.pandas:
             return [numpy.nan if isinstance(element, basestring) and element == ' ' else element for element in list]
         else:
             return list