Merge branch 'feature/tabular-size-limit' into develop

 [SVCS-675]
 Closes: #325

Author: felliott

mfr/extensions/tabular/exceptions.py

@@ -29,9 +29,14 @@ class TableTooBigError(TabularRendererError):
 
     __TYPE = 'tabular_table_too_big'
 
-    def __init__(self, message, *args, code: int=400, **kwargs):
+    def __init__(self, message, *args, code: int=400, nbr_cols: int=0, nbr_rows: int=0, **kwargs):
         super().__init__(message, *args, code=code, **kwargs)
-        self.attr_stack.append([self.__TYPE, {}])
+        self.nbr_cols = nbr_cols
+        self.nbr_rows = nbr_rows
+        self.attr_stack.append([self.__TYPE, {
+            'nbr_cols': self.nbr_cols,
+            'nbr_rows': self.nbr_rows
+        }])
 
 
 class UnexpectedFormattingError(TabularRendererError):
@@ -42,3 +47,20 @@ def __init__(self, message, *args, code: int=500, formatting_function: str='', *
         super().__init__(message, *args, code=code, **kwargs)
         self.formatting_function = formatting_function
         self.attr_stack.append([self.__TYPE, {'formatting_function': self.formatting_function}])
+
+
+class FileTooLargeError(TabularRendererError):
+
+    __TYPE = 'tabular_file_too_large'
+
+    def __init__(self, message, *args, code: int=400, file_size: int=None, max_size: int=None,
+                 **kwargs):
+        super().__init__(message, *args, code=code, **kwargs)
+
+        self.file_size = file_size
+        self.max_size = max_size
+
+        self.attr_stack.append([self.__TYPE, {
+            'file_size': self.file_size,
+            'max_size': self.max_size
+        }])

mfr/extensions/tabular/libs/xlrd_tools.py

@@ -19,7 +19,8 @@ def xlsx_xlrd(fp):
 
     for sheet in wb.sheets():
         if sheet.ncols > max_size or sheet.nrows > max_size:
-            raise TableTooBigError('Table is too large to render.', '.xlsx')
+            raise TableTooBigError('Table is too large to render.', '.xlsx',
+                                   nbr_cols=sheet.ncols, nbr_rows=sheet.nrows)
 
         if sheet.ncols < 1 or sheet.nrows < 1:
             sheets[sheet.name] = ([], [])

mfr/extensions/tabular/render.py

@@ -1,11 +1,14 @@
-import os
 import json
+import logging
+import os
 
+from humanfriendly import format_size
 from mako.lookup import TemplateLookup
+
 from mfr.core import extension
+from mfr.extensions.tabular import settings, exceptions
 
-from mfr.extensions.tabular import settings
-from mfr.extensions.tabular import exceptions
+logger = logging.getLogger(__name__)
 
 
 class TabularRenderer(extension.BaseRenderer):
@@ -16,6 +19,16 @@ class TabularRenderer(extension.BaseRenderer):
         ]).get_template('viewer.mako')
 
     def render(self):
+        file_size = os.path.getsize(self.file_path)
+        if file_size > settings.MAX_FILE_SIZE:
+            raise exceptions.FileTooLargeError(
+                'Tabular files larger than {} are not rendered. Please download '
+                'the file to view.'.format(format_size(settings.MAX_FILE_SIZE, binary=True)),
+                file_size=file_size,
+                max_size=settings.MAX_FILE_SIZE,
+                extension=self.metadata.ext,
+            )
+
         with open(self.file_path, errors='replace') as fp:
             sheets, size = self._render_grid(fp, self.metadata.ext)
             return self.TEMPLATE.render(
@@ -34,7 +47,7 @@ def file_required(self):
     def cache_result(self):
         return True
 
-    def _render_grid(self, fp, ext, *args, **kwargs):  # assets_path, ext):
+    def _render_grid(self, fp, ext, *args, **kwargs):
         """Render a tabular file to html
         :param fp: file pointer object
         :return: RenderResult object containing html and assets
@@ -46,22 +59,31 @@ def _render_grid(self, fp, ext, *args, **kwargs):  # assets_path, ext):
         size = settings.SMALL_TABLE
         self._renderer_tabular_metrics['size'] = 'small'
         self._renderer_tabular_metrics['nbr_sheets'] = len(sheets)
-        for sheet in sheets:
-            sheet = sheets[sheet]  # Sheets are stored in key-value pairs of the form {sheet: (col, row)}
-            if len(sheet[0]) > 9:  # Check the number of columns
+        for sheet_title in sheets:
+            sheet = sheets[sheet_title]
+
+            # sheet is a two-element list.  sheet[0] is a list of dicts containing metadata about
+            # the column headers.  Each dict contains four keys: `field`, `name`, `sortable`, `id`.
+            # sheet[1] is a list of dicts where each dict contains the row data.  The keys are the
+            # fields the data belongs to and the values are the data values.
+
+            nbr_cols = len(sheet[0])
+            if nbr_cols > 9:
                 size = settings.BIG_TABLE
                 self._renderer_tabular_metrics['size'] = 'big'
 
-            if len(sheet[0]) > settings.MAX_SIZE or len(sheet[1]) > settings.MAX_SIZE:
-                raise exceptions.TableTooBigError('Table is too large to render.', extension=ext)
+            nbr_rows = len(sheet[1])
+            if nbr_cols > settings.MAX_SIZE or nbr_rows > settings.MAX_SIZE:
+                raise exceptions.TableTooBigError('Table is too large to render.', extension=ext,
+                                                  nbr_cols=nbr_cols, nbr_rows=nbr_rows)
 
         return sheets, size
 
     def _populate_data(self, fp, ext):
         """Determine the appropriate library and use it to populate rows and columns
         :param fp: file pointer
         :param ext: file extension
-        :return: tuple of column headers and row data
+        :return: a dict mapping sheet titles to tuples of column headers and row data
         """
         function_preference = settings.LIBS.get(ext.lower())
 

mfr/extensions/tabular/settings.py

@@ -4,7 +4,8 @@
 
 config = settings.child('TABULAR_EXTENSION_CONFIG')
 
-MAX_SIZE = int(config.get('MAX_SIZE', 10000))
+MAX_FILE_SIZE = int(config.get('MAX_FILE_SIZE', 10 * 1024 * 1024))  # 10Mb
+MAX_SIZE = int(config.get('MAX_SIZE', 10000))  # max number of rows or columns allowed.
 TABLE_WIDTH = int(config.get('TABLE_WIDTH', 700))
 TABLE_HEIGHT = int(config.get('TABLE_HEIGHT', 600))