add zip rendering and tests

Johnetordoff · felliott · commit fd602f928cfe · 2017-11-10T12:18:17.000-05:00
* Render .zip files as a listing of the contents.  Each entry
   includes the name, timestamp, and size of the file.  macOS-specific
   directories are excluded from the listing.

 [SVCS-248]
diff --git a/dev-requirements.txt b/dev-requirements.txt
@@ -1,6 +1,7 @@
 -r requirements.txt
 
 -e git+https://github.com/centerforopenscience/aiohttpretty.git@0.0.2#egg=aiohttpretty
+beautifulsoup4
 colorlog==2.5.0
 flake8==3.0.4
 ipdb
diff --git a/mfr/core/utils.py b/mfr/core/utils.py
@@ -98,6 +98,9 @@ def make_renderer(name, metadata, file_path, url, assets_url, export_url):
         )
 
 def sizeof_fmt(num, suffix='B'):
+    if abs(num) < 1000:
+        return '%3.0f%s' % (num, suffix)
+
     for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']:
         if abs(num) < 1000.0:
             return '%3.1f%s%s' % (num, unit, suffix)
diff --git a/mfr/extensions/zip/__init__.py b/mfr/extensions/zip/__init__.py
@@ -0,0 +1 @@
+from .render import ZipRenderer  # noqa
diff --git a/mfr/extensions/zip/render.py b/mfr/extensions/zip/render.py
@@ -0,0 +1,36 @@
+import os
+import zipfile
+
+import markupsafe
+from mako.lookup import TemplateLookup
+
+from mfr.core import extension
+from mfr.core.utils import sizeof_fmt
+
+
+class ZipRenderer(extension.BaseRenderer):
+
+    TEMPLATE = TemplateLookup(
+        directories=[
+            os.path.join(os.path.dirname(__file__), 'templates')
+        ]).get_template('viewer.mako')
+
+    def render(self):
+        zip_file = zipfile.ZipFile(self.file_path, 'r')
+
+        filelist = [{'name': markupsafe.escape(file.filename),
+                     'size': sizeof_fmt(int(file.file_size)),
+                     'date': "%d-%02d-%02d %02d:%02d:%02d" % file.date_time[:6]} for file in zip_file.filelist
+                    if not file.filename.startswith('__MACOSX')]
+
+        message = '' if filelist else 'This zip file is empty.'
+
+        return self.TEMPLATE.render(zipped_filenames=filelist, message=message)
+
+    @property
+    def file_required(self):
+        return True
+
+    @property
+    def cache_result(self):
+        return True
diff --git a/mfr/extensions/zip/templates/viewer.mako b/mfr/extensions/zip/templates/viewer.mako
@@ -0,0 +1,31 @@
+<link href="https://fonts.googleapis.com/css?family=Open+Sans:400,600,300,700" rel="stylesheet" type="text/css">
+<link rel="stylesheet" href="static/css/bootstrap.min.css">
+<link rel="stylesheet" href="static/css/default.css">
+<div style="word-wrap: break-word;" class="mfrViewer">
+    <h1>
+        Zip File:
+    </h1>
+    <p>
+        Download the .zip file to view the contents.
+    </p>
+    ${message}
+    <table class="table table-hover">
+        <thead>
+            <th>File Name</th>
+            <th>Modified</th>
+            <th>Size</th>
+        </thead>
+        <tbody>
+            % for file in zipped_filenames:
+                <tr>
+                    <td>${file['name']}</td>
+                    <td>${file['date']}</td>
+                    <td>${file['size']}<td>
+                </tr>
+            % endfor
+        </tbody>
+    </table>
+</div>
+
+<script src="/static/js/mfr.js"></script>
+<script src="/static/js/mfr.child.js"></script>
diff --git a/mfr/settings.py b/mfr/settings.py
@@ -85,7 +85,12 @@ def child(self, key):
 UNSUPPORTED_RENDER_MSG = 'Viewing of this file type is not currently supported. Please download the file to view.'
 
 
-MAX_FILE_SIZE_TO_RENDER = {'.csv': 100000000, '.xlsx': 1000000000, '.tsv': 1000000000, '.sav': 1000000000, '.xls': 1000000000}
+MAX_FILE_SIZE_TO_RENDER = {'.csv': 100000000,
+                           '.xlsx': 1000000000,
+                           '.tsv': 1000000000,
+                           '.sav': 1000000000,
+                           '.xls': 1000000000,
+                           '.zip': 1000000000}
 
 try:
     import colorlog  # noqa
diff --git a/setup.py b/setup.py
@@ -791,7 +791,10 @@ def parse_requirements(requirements):
             '.webm = mfr.extensions.video:VideoRenderer',
 
             # JASP
-            '.jasp = mfr.extensions.jasp:JASPRenderer'
+            '.jasp = mfr.extensions.jasp:JASPRenderer',
+
+            # Zip
+            '.zip = mfr.extensions.zip:ZipRenderer'
         ]
     },
 )
diff --git a/tests/extensions/zip/__init__.py b/tests/extensions/zip/__init__.py
diff --git a/tests/extensions/zip/files/empty.zip b/tests/extensions/zip/files/empty.zip
diff --git a/tests/extensions/zip/files/test.zip b/tests/extensions/zip/files/test.zip
diff --git a/tests/extensions/zip/test_renderer.py b/tests/extensions/zip/test_renderer.py
@@ -0,0 +1,90 @@
+import os
+import re
+from zipfile import ZipFile
+
+import pytest
+from bs4 import BeautifulSoup
+
+from mfr.core.provider import ProviderMetadata
+from mfr.extensions.zip import ZipRenderer
+
+
+BASE = os.path.dirname(os.path.abspath(__file__))
+
+
+@pytest.fixture
+def metadata():
+    return ProviderMetadata('test',
+                            '.zip',
+                            'text/plain',
+                            '1234',
+                            'http://wb.osf.io/file/test.zip?token=1234')
+
+@pytest.fixture
+def zip_file():
+    return ZipFile(os.path.join(BASE, 'files', 'test.zip'), 'r')
+
+
+@pytest.fixture
+def zip_empty_file():
+    return ZipFile(os.path.join(BASE, 'files', 'empty.zip'), 'r')
+
+
+@pytest.fixture
+def test_file_path():
+    return os.path.join(BASE, 'files', 'test.zip')
+
+
+@pytest.fixture
+def url():
+    return 'http://osf.io/file/test.zip'
+
+
+@pytest.fixture
+def assets_url():
+    return 'http://mfr.osf.io/assets'
+
+
+@pytest.fixture
+def export_url(url):
+    return 'http://mfr.osf.io/export?url=' + url
+
+
+@pytest.fixture
+def renderer(metadata, test_file_path, url, assets_url, export_url):
+    return ZipRenderer(metadata, test_file_path, url, assets_url, export_url)
+
+
+
+def remove_whitespace(str):
+    str = re.sub('\n*', '', str)
+    return re.sub(r'\ {2,}', '', str)
+
+
+class TestZipRenderer:
+
+    def test_render(self, renderer):
+        body = renderer.render()
+        parsed_html = BeautifulSoup(body)
+        rows = parsed_html.findChildren('table')[0].findChildren(['tr'])
+
+        name = rows[2].findChildren('td')[0].get_text().strip()
+        assert 'test/test 1' == name
+
+        date_modified = rows[2].findChildren('td')[1].get_text().strip()
+        assert '2017-03-02 16:22:14' == date_modified
+
+        size = rows[2].findChildren('td')[2].get_text().strip()
+        assert '15B' == size
+
+        # non-expanded zip file should have no children
+        name = rows[4].findChildren('td')[0].get_text().strip()
+        assert 'test/zip file which is not expanded.zip' == name
+        assert body.count('zip file which is not expanded.zip') == 1
+
+        size = rows[4].findChildren('td')[2].get_text().strip()
+        assert '1.8KB' == size  # formatting of larger byte sizes
+
+        # hidden files should be hidden
+        assert body.count('__MACOSX') == 0
+

Original file line number	Diff line number	Diff line change
`@@ -98,6 +98,9 @@ def make_renderer(name, metadata, file_path, url, assets_url, export_url):`
`98`	`98`	`)`
`99`	`99`
`100`	`100`	`def sizeof_fmt(num, suffix='B'):`
	`101`	`+ if abs(num) < 1000:`
	`102`	`+ return '%3.0f%s' % (num, suffix)`
	`103`	`+`
`101`	`104`	`for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']:`
`102`	`105`	`if abs(num) < 1000.0:`
`103`	`106`	`return '%3.1f%s%s' % (num, unit, suffix)`
Original file line number	Diff line number	Diff line change
`@@ -791,7 +791,10 @@ def parse_requirements(requirements):`
`791`	`791`	`'.webm = mfr.extensions.video:VideoRenderer',`
`792`	`792`
`793`	`793`	`# JASP`
`794`		`- '.jasp = mfr.extensions.jasp:JASPRenderer'`
	`794`	`+ '.jasp = mfr.extensions.jasp:JASPRenderer',`
	`795`	`+`
	`796`	`+ # Zip`
	`797`	`+ '.zip = mfr.extensions.zip:ZipRenderer'`
`795`	`798`	`]`
`796`	`799`	`},`
`797`	`800`	`)`