Merge branch 'release/0.25.0'

Author: felliott

.travis.yml

@@ -15,7 +15,7 @@ before_install:
 
 install:
   - travis_retry pip install --upgrade pip
-  - travis_retry pip install setuptools==30.4.0
+  - travis_retry pip install setuptools==37.0.0
   - travis_retry pip install wheel==0.26.0
   - travis_retry pip install invoke==0.13.0
   - travis_retry invoke wheelhouse --develop

CHANGELOG

@@ -2,6 +2,22 @@
 ChangeLog
 *********
 
+0.25.0 (2018-03-28)
+===================
+- Feature: Large files can now be zoomed!  Images are scaled to fit the window, but clicking on the
+image will show a higher-resolution version.  The mousewheel or two-finger drag will zoom in up to
+the maximum image size of 2400x2400px (increased from 1200x1200px).  The zoomed image can be panned
+around the display port by moving the mouse.
+- Feature: Refuse to render tabular files (.xls, .xlsx, .csv) larger than 10Mb. Rendering these
+files consumes a lot of memory and can crash MFR.
+- Feature: Cached files now have the renderer that generated them appended to their names.  This
+allows DevOps to do more targeted cache-cleaning after a feature release.
+- Fix: Make Libreoffice download mirror url configurable in the Dockerfile.  The current mirror
+was having difficulty serving requests for awhile.  This makes it easier to switch mirrors during
+times of trouble.  The official mirror is working again, and MFR will continue to use that.
+- Code: Update MFR to support setuptools versions greater than v30.4.0. MFR's `__version__`
+declaration has moved from `mfr.__init__` to `mfr.version`.  (thanks, @johnetordoff!)
+
 0.24.2 (2018-02-09)
 ===================
 - Fix: Update UNOCONV_BIN setting to reflect the default pip install path.

Dockerfile

@@ -43,19 +43,18 @@ RUN usermod -d /home www-data \
         curl \
     && rm -rf /var/lib/apt/lists/* \
     && pip install -U pip \
-    && pip uninstall -y setuptools \
-    && rm -f /usr/local/lib/python3.5/site-packages/mfr-nspkg.pth \
-    && pip install setuptools==30.4.0 \
+    && pip install setuptools==37.0.0 \
     && mkdir -p /code
 
-ENV LIBREOFFICE_VERSION 6.0.1.1
-ENV LIBREOFFICE_ARCHIVE LibreOffice_6.0.1.1_Linux_x86-64_deb.tar.gz
+ENV LIBREOFFICE_VERSION 6.0.2.1
+ENV LIBREOFFICE_ARCHIVE LibreOffice_6.0.2.1_Linux_x86-64_deb.tar.gz
+ENV LIBREOFFICE_MIRROR_URL https://downloadarchive.documentfoundation.org/libreoffice/old/
 RUN apt-get update \
     && apt-get install -y \
         curl \
     && gpg --keyserver pool.sks-keyservers.net --recv-keys AFEEAEA3 \
-    && curl -SL "https://downloadarchive.documentfoundation.org/libreoffice/old/$LIBREOFFICE_VERSION/deb/x86_64/$LIBREOFFICE_ARCHIVE" -o $LIBREOFFICE_ARCHIVE \
-    && curl -SL "https://downloadarchive.documentfoundation.org/libreoffice/old/$LIBREOFFICE_VERSION/deb/x86_64/$LIBREOFFICE_ARCHIVE.asc" -o $LIBREOFFICE_ARCHIVE.asc \
+    && curl -SL "$LIBREOFFICE_MIRROR_URL/$LIBREOFFICE_VERSION/deb/x86_64/$LIBREOFFICE_ARCHIVE" -o $LIBREOFFICE_ARCHIVE \
+    && curl -SL "$LIBREOFFICE_MIRROR_URL/$LIBREOFFICE_VERSION/deb/x86_64/$LIBREOFFICE_ARCHIVE.asc" -o $LIBREOFFICE_ARCHIVE.asc \
     && gpg --verify "$LIBREOFFICE_ARCHIVE.asc" \
     && mkdir /tmp/libreoffice \
     && tar -xvf "$LIBREOFFICE_ARCHIVE" -C /tmp/libreoffice/ --strip-components=1 \

README.md

@@ -41,8 +41,10 @@ After installing python3.5, create the virtual environment with the following co
 pip install virtualenv
 pip install virtualenvwrapper
 mkvirtualenv --python=`which python3.5` mfr
-pip install setuptools==30.4.0
+
+pip install setuptools==37.0.0
 pip install invoke==0.13.0
+
 invoke install
 invoke server
 ```
@@ -86,8 +88,7 @@ invoke test
 
 ### Known issues
 
-- Running `invoke install -d` with setuptools v31 or greater can break MFR.  The symptom error message is: `"AttributeError: module 'mfr' has no attribute '__version__'".`  If you encounter this, you will need to remove the file
-`mfr-nspkg.pth` from your virtualenv directory, run `pip install setuptools==30.4.0`, then re-run `invoke install -d`.
+- **Updated, 2018-03-01:** *MFR has been updated to work with setuptools==37.0.0 as of MFR release v0.25. The following issue should not happen for new installs, but may occur if you downgrade to an older version.* Running `invoke install -d` with setuptools v31 or greater can break MFR.  The symptom error message is: `"AttributeError: module 'mfr' has no attribute '__version__'".`  If you encounter this, you will need to remove the file `mfr-nspkg.pth` from your virtualenv directory, run `pip install setuptools==30.4.0`, then re-run `invoke install -d`.
 
 - The error `def create_default_context(purpose=ssl.Purpose.SERVER_AUTH, *, cafile=None, capath=None, cadata=None): SyntaxError: invalid syntax` is caused by inadvertently running the wrong version of Python. This can be caused by hashing the alias `inv`. To fix this run the command `hash -d inv` then run `inv server`.
 
@@ -99,7 +100,7 @@ Interested in adding support for a new provider or file format? Check out the CO
 
 ### License
 
-Copyright 2013-2017 Center for Open Science
+Copyright 2013-2018 Center for Open Science
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

dev-requirements.txt

@@ -1,6 +1,8 @@
 -r requirements.txt
 
--e git+https://github.com/centerforopenscience/aiohttpretty.git@0.0.2#egg=aiohttpretty
+# aiohttpretty disabled due to no usage in MFR, re-enable when needed
+# git+https://github.com/centerforopenscience/aiohttpretty.git@0.1.0#egg=aiohttpretty
+
 beautifulsoup4
 colorlog==2.5.0
 flake8==3.0.4

docs/conf.py

@@ -1,23 +1,25 @@
 # -*- coding: utf-8 -*-
 #
-# mfr documentation build configuration file.
+# MFR documentation build configuration file.
 #
-# This file is execfile()d with the current directory set to its containing dir.
+# This file is generated by execfile() with the current directory set to its
+# containing dir.
 #
-# Note that not all possible configuration values are present in this
-# autogenerated file.
+# Note that not all possible configuration values are present in this auto-
+# generated file.
 #
 # All configuration values have a default; values that are commented out
 # serve to show the default.
 
 import sys
 import os
 
+from mfr.version import __version__
+
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 sys.path.insert(0, os.path.abspath('..'))
-import mfr  # noqa
 sys.path.append(os.path.abspath("_themes"))
 
 # -- General configuration -----------------------------------------------------
@@ -45,14 +47,14 @@
 
 # General information about the project.
 project = u'mfr'
-copyright = u'2017, Center For Open Science'
+copyright = u'2018, Center For Open Science'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = release = mfr.__version__
+version = release = __version__
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/install.rst

@@ -20,7 +20,7 @@ Install the versions of ``setuptools`` and ``invoke`` found in the requirements.
 
 .. code-block:: bash
 
-    pip install setuptools==30.4.0
+    pip install setuptools==37.0.0
     pip install invoke==0.13.0
 
 Install requirements:

mfr/__init__.py

@@ -1,2 +1,4 @@
-__version__ = '0.24.2'
+# This is a namespace package, don't put any functional code in here besides the
+# declare_namespace call, or it will disappear on install. See:
+# https://setuptools.readthedocs.io/en/latest/setuptools.html#namespace-packages
 __import__('pkg_resources').declare_namespace(__name__)

mfr/core/remote_logging.py

@@ -3,12 +3,9 @@
 import logging
 
 import aiohttp
-# from geoip import geolite2
 
-import mfr
 from mfr.server import settings
-
-# import waterbutler
+from mfr.version import __version__
 from waterbutler.core.utils import async_retry
 
 
@@ -22,8 +19,7 @@ async def log_analytics(request, metrics, is_error=False):
 
     keen_payload = copy.deepcopy(metrics)
     keen_payload['meta'] = {
-        'mfr_version': mfr.__version__,
-        # 'wb_version': waterbutler.__version__,
+        'mfr_version': __version__,
         'epoch': 1,
     }
     keen_payload.update(request)

mfr/core/utils.py

@@ -1,3 +1,4 @@
+import pkg_resources
 from stevedore import driver
 
 from mfr.core import exceptions
@@ -97,6 +98,48 @@ def make_renderer(name, metadata, file_path, url, assets_url, export_url):
             }
         )
 
+
+def get_renderer_name(name: str) -> str:
+    """ Return the name of the renderer used for a certain file extension.
+
+    :param str name: The name of the extension to get the renderer name for. (.jpg, .docx, etc)
+
+    :rtype : `str`
+    """
+
+    # `ep_iterator` is an iterable object. Must convert it to a `list` for access.
+    # `list()` can only be called once because the iterator moves to the end after conversion.
+    ep_iterator = pkg_resources.iter_entry_points(group='mfr.renderers', name=name.lower())
+    ep_list = list(ep_iterator)
+
+    # Empty list indicates unsupported file type.
+    # Return a blank string and let `make_renderer()` handle it.
+    if len(ep_list) == 0:
+        return ''
+
+    # If file type is supported, there must be only one element in the list.
+    assert len(ep_list) == 1
+    return ep_list[0].attrs[0]
+
+
+def get_exporter_name(name: str) -> str:
+    """ Return the name of the exporter used for a certain file extension.
+
+    :param str name: The name of the extension to get the exporter name for. (.jpg, .docx, etc)
+
+    :rtype : `str`
+    """
+
+    # `ep_iterator` is an iterable object. Must convert it to a `list` for access.
+    # `list()` can only be called once because the iterator moves to the end after conversion.
+    ep_iterator = pkg_resources.iter_entry_points(group='mfr.exporters', name=name.lower())
+    ep_list = list(ep_iterator)
+
+    # `make_renderer()` is called before `make_exporter()` to ensure the file type is supported
+    assert len(ep_list) == 1
+    return ep_list[0].attrs[0]
+
+
 def sizeof_fmt(num, suffix='B'):
     if abs(num) < 1000:
         return '%3.0f%s' % (num, suffix)