Add support for .dta to tabular renderer

 * Also added notes to some commented out formats in setup.py

 [SVCS-403]

Author: AddisonSchiller

mfr/extensions/tabular/libs/panda_tools.py

@@ -33,7 +33,7 @@ def dta_pandas(fp):
     :param fp: File pointer object
     :return: tuple of table headers and data
     """
-    dataframe = pandas.read_stata(fp)
+    dataframe = pandas.read_stata(fp.name).replace(numpy.nan, '')
     return data_from_dataframe(dataframe)
 
 

mfr/extensions/tabular/render.py

@@ -74,7 +74,7 @@ def _populate_data(self, fp, ext):
                 self._renderer_tabular_metrics['importer'] = function.__name__
                 try:
                     return imported(fp)
-                except KeyError:
+                except (KeyError, ValueError):
                     raise exceptions.UnexpectedFormattingError(
                         'Unexpected formatting error.',
                         extension=self.metadata.ext,

setup.py

@@ -113,7 +113,8 @@ def parse_requirements(requirements):
             '.stw = mfr.extensions.unoconv:UnoconvExporter',
             '.svg = mfr.extensions.unoconv:UnoconvExporter',
             '.svm = mfr.extensions.unoconv:UnoconvExporter',
-            '.swf = mfr.extensions.unoconv:UnoconvExporter',
+            # Renders junk at the moment
+            # '.swf = mfr.extensions.unoconv:UnoconvExporter',
             '.sxc = mfr.extensions.unoconv:UnoconvExporter',
             '.sxd = mfr.extensions.unoconv:UnoconvExporter',
             '.sxi = mfr.extensions.unoconv:UnoconvExporter',
@@ -704,7 +705,7 @@ def parse_requirements(requirements):
             '.tsv = mfr.extensions.tabular:TabularRenderer',
             '.xls = mfr.extensions.tabular:TabularRenderer',
             '.xlsx = mfr.extensions.tabular:TabularRenderer',
-            #'.dta = mfr.extensions.tabular:TabularRenderer',
+            '.dta = mfr.extensions.tabular:TabularRenderer',
             '.sav = mfr.extensions.tabular:TabularRenderer',
             #'.ods = mfr.extensions.tabular:TabularRenderer',
 
@@ -764,7 +765,8 @@ def parse_requirements(requirements):
             '.stw = mfr.extensions.unoconv:UnoconvRenderer',
             '.svg = mfr.extensions.unoconv:UnoconvRenderer',
             '.svm = mfr.extensions.unoconv:UnoconvRenderer',
-            '.swf = mfr.extensions.unoconv:UnoconvRenderer',
+            # Renders junk at the moment
+            # '.swf = mfr.extensions.unoconv:UnoconvRenderer',
             '.sxc = mfr.extensions.unoconv:UnoconvRenderer',
             '.sxd = mfr.extensions.unoconv:UnoconvRenderer',
             '.sxi = mfr.extensions.unoconv:UnoconvRenderer',
@@ -787,6 +789,7 @@ def parse_requirements(requirements):
             # video
             '.mp4 = mfr.extensions.video:VideoRenderer',
             '.m4v = mfr.extensions.video:VideoRenderer',
+            # Unable to support at this time
             #'.avi = mfr.extensions.video:VideoRenderer',
             '.ogv = mfr.extensions.video:VideoRenderer',
             #'.wmv = mfr.extensions.video:VideoRenderer',

tests/extensions/tabular/files/invalid.dta

@@ -0,0 +1,8 @@
+,id,target,gender,likely_discriminated,promoted,responsible,compensated,would_apply,chances,attractive_employer,put_off_similar_jobs,work,work_experience,studies,age
+0,159.0,Erik,male,4.0,4.0,4.0,1.0,6.0,6.0,1.0,1.0,studier,mattlig,Psykologi,22.0
+1,97.0,Maria,male,5.0,4.0,6.0,4.0,4.0,4.0,2.0,1.0,studier,stor,Elektroteknik,25.0
+2,56.0,Erik,male,1.0,4.0,1.0,1.0,5.0,1.0,4.0,1.0,studier,lite,Medicin,23.0
+3,205.0,Maria,male,1.0,4.0,7.0,1.0,7.0,1.0,4.0,1.0,studier,mattlig,ekonomi och ledarskap,25.0
+4,206.0,Erik,male,1.0,1.0,1.0,1.0,7.0,4.0,4.0,1.0,studier,mattlig,miljö,25.0
+5,210.0,Erik,male,1.0,1.0,1.0,1.0,7.0,4.0,4.0,1.0,arbete,mattlig, ,34.0
+6,151.0,Maria,male,1.0,4.0,2.0,1.0,7.0,4.0,4.0,1.0,studier,mattlig,Strat komm,24.0

tests/extensions/tabular/files/test.dta

@@ -1,5 +1,7 @@
 import os
 
+import pytest
+
 from mfr.extensions.tabular.libs import panda_tools
 
 BASE = os.path.dirname(os.path.abspath(__file__))
@@ -31,11 +33,35 @@ def test_tsv_pandas(self):
         assert sheet[0][0] == {'field': 'one\ttwo\tthree', 'id': 'one\ttwo\tthree', 'name': 'one\ttwo\tthree', 'sortable': True}
         assert sheet[1][0] == {'one\ttwo\tthree': 'a\tb\tc'}
 
-    # def test_dta_pandas():
-    #     with open('mfr_tabular/tests/fixtures/test.dta') as fp:
-    #         headers, data = panda_tools.dta_pandas(fp)
+    def test_invalid_dta(self):
+        with open(os.path.join(BASE, 'files', 'invalid.dta')) as fp:
+            with pytest.raises(ValueError):
+                panda_tools.dta_pandas(fp)
+
+    def test_dta_pandas(self):
+        with open(os.path.join(BASE, 'files', 'test.dta')) as fp:
+            sheets = panda_tools.dta_pandas(fp)
 
-    #     assert headers[0] == {'field': 'one', 'id': 'one', 'name': 'one'}
-    #     assert data[0] == {'one': 'a', 'two': 'b', 'three': 'c'}
-    #     assert len(data) is 2
-    #     assert len(headers) is 3
+        sheet = sheets.popitem()[1]
+        assert sheet == ([{'id': 'Actor', 'name': 'Actor', 'field': 'Actor', 'sortable': True},
+            {'id': 'IMDBscore', 'name': 'IMDBscore', 'field': 'IMDBscore', 'sortable': True}],
+            [{'Actor': '1', 'IMDBscore': 6.6}, {'Actor': '1', 'IMDBscore': 6.5},
+            {'Actor': '1', 'IMDBscore': 7.3}, {'Actor': '1', 'IMDBscore': 7.8},
+            {'Actor': '1', 'IMDBscore': 6.3}, {'Actor': '1', 'IMDBscore': 6.6},
+            {'Actor': '1', 'IMDBscore': 6.1}, {'Actor': '1', 'IMDBscore': 7.1},
+            {'Actor': '1', 'IMDBscore': 5.7}, {'Actor': '1', 'IMDBscore': 5.7},
+            {'Actor': '1', 'IMDBscore': 7.1}, {'Actor': '1', 'IMDBscore': 7.6},
+            {'Actor': '1', 'IMDBscore': 7.1}, {'Actor': '1', 'IMDBscore': 3.8},
+            {'Actor': '1', 'IMDBscore': 8.0}, {'Actor': '1', 'IMDBscore': 4.7},
+            {'Actor': '1', 'IMDBscore': 6.5}, {'Actor': '1', 'IMDBscore': 7.6},
+            {'Actor': '1', 'IMDBscore': 6.3}, {'Actor': '1', 'IMDBscore': 5.4},
+            {'Actor': '2', 'IMDBscore': 7.1}, {'Actor': '2', 'IMDBscore': 7.3},
+            {'Actor': '2', 'IMDBscore': 6.2}, {'Actor': '2', 'IMDBscore': 7.4},
+            {'Actor': '2', 'IMDBscore': 7.5}, {'Actor': '2', 'IMDBscore': 6.7},
+            {'Actor': '2', 'IMDBscore': 6.2}, {'Actor': '2', 'IMDBscore': 4.7},
+            {'Actor': '2', 'IMDBscore': 6.5}, {'Actor': '2', 'IMDBscore': 5.5},
+            {'Actor': '2', 'IMDBscore': 5.4}, {'Actor': '2', 'IMDBscore': 7.7},
+            {'Actor': '2', 'IMDBscore': 7.1}, {'Actor': '2', 'IMDBscore': 6.1},
+            {'Actor': '2', 'IMDBscore': 6.7}, {'Actor': '2', 'IMDBscore': 7.0},
+            {'Actor': '2', 'IMDBscore': 5.8}, {'Actor': '2', 'IMDBscore': 4.1},
+            {'Actor': '2', 'IMDBscore': 5.7}, {'Actor': '2', 'IMDBscore': 6.1}])