Merge tag '0.25.3' into develop

Author: felliott

CHANGELOG

@@ -2,6 +2,12 @@
 ChangeLog
 *********
 
+0.25.3 (2018-04-03)
+===================
+- Fix: Add a subprocess timeout to the unoconv exporter so MFR doesn't wait forever for a process
+that might not complete.
+- Feature: Add user-agent to tornado access logs to help identify spiders.
+
 0.25.2 (2018-03-29)
 ===================
 - Fix: Release memory consumed during csv-rendering process and force a garbage-collect cycle to

mfr/extensions/unoconv/export.py

@@ -11,15 +11,15 @@ class UnoconvExporter(extension.BaseExporter):
 
     def export(self):
         try:
-            subprocess.check_call([
+            subprocess.run([
                 settings.UNOCONV_BIN,
                 '-n',
                 '-c', 'socket,host={},port={};urp;StarOffice.ComponentContext'.format(settings.ADDRESS, settings.PORT),
                 '-f', self.format,
                 '-o', self.output_file_path,
                 '-vvv',
                 self.source_file_path
-            ])
+            ], check=True, timeout=settings.UNOCONV_TIMEOUT)
         except subprocess.CalledProcessError as err:
             name, extension = os.path.splitext(os.path.split(self.source_file_path)[-1])
             raise exceptions.SubprocessError(

mfr/extensions/unoconv/settings.py

@@ -6,6 +6,7 @@
 config = settings.child('UNOCONV_EXTENSION_CONFIG')
 
 UNOCONV_BIN = config.get('UNOCONV_BIN', '/usr/local/bin/unoconv')
+UNOCONV_TIMEOUT = int(config.get('UNOCONV_TIMEOUT', 60))
 
 ADDRESS = config.get('SERVER', os.environ.get('UNOCONV_PORT_2002_TCP_ADDR', '127.0.0.1'))
 PORT = config.get('PORT', os.environ.get('UNOCONV_PORT_2002_TCP_PORT', '2002'))

mfr/server/app.py

@@ -1,3 +1,4 @@
+import time
 import signal
 import asyncio
 import logging
@@ -19,6 +20,7 @@
 from mfr.version import __version__
 
 logger = logging.getLogger(__name__)
+access_logger = logging.getLogger('tornado.access')
 
 
 def sig_handler(sig, frame):
@@ -33,6 +35,16 @@ def stop_loop():
     io_loop.add_callback_from_signal(stop_loop)
 
 
+def almost_apache_style_log(handler):
+    '''without status code and body length'''
+    req = handler.request
+    access_logger.info('%s - - [%s +0800] "%s %s %s" - - "%s" "%s"' %
+                       (req.remote_ip, time.strftime("%d/%b/%Y:%X"), req.method,
+                        req.uri,
+                        req.version, getattr(req, 'referer', '-'),
+                        req.headers['User-Agent']))
+
+
 def make_app(debug):
     app = tornado.web.Application(
         [
@@ -45,6 +57,7 @@ def make_app(debug):
             (r'/status', StatusHandler),
         ],
         debug=debug,
+        log_function=almost_apache_style_log,
     )
     app.sentry_client = AsyncSentryClient(settings.SENTRY_DSN, release=__version__)
     return app

mfr/server/handlers/core.py

@@ -203,8 +203,10 @@ def log_exception(self, typ, value, tb):
                         list(value.args))
                 tornado.web.gen_log.warning(format, *args)
         else:
-            tornado.web.app_log.error("Uncaught exception %s\n", self._request_summary(),
-                                       exc_info=(typ, value, tb))
+            tornado.web.app_log.error("[User-Agent: %s] Uncaught exception %s\n",
+                                      self.request.headers.get('User-Agent', '*none found*'),
+                                      self._request_summary(),
+                                      exc_info=(typ, value, tb))
 
     def on_finish(self):
         if self.request.method not in self.ALLOWED_METHODS:

mfr/version.py

@@ -1 +1 @@
-__version__ = '0.25.2'
+__version__ = '0.25.3'